{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.24.3\n",
      "2.0.3\n"
     ]
    }
   ],
   "source": [
    "print(np.__version__)\n",
    "print(pd.__version__)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.set_option('display.max_columns', None)\n",
    "pd.set_option('display.max_rows', None)\n",
    "pd.set_option('display.max_colwidth', 10000)\n",
    "pd.set_option('display.width', 10000)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 以下参数目前需要手动改写，暂时无法从profile后的文件中获取\n",
    "# 指定通过tpu_profile.py --format layer生成的profile文件路径\n",
    "profile_dir = '/workspace/jira/VIT/int8_b16/test/'\n",
    "# TPU的频率\n",
    "tpu_freq = 1e9 # 1000MHz\n",
    "# ModelAlgOps来自final.mlir中，这是模型原始的计算量，profile中的AlgOps是原本算子拆成多个指令后的，指令的有效计算量，与实现相关\n",
    "ModelAlgOps = 564150211456\n",
    "# 端到端runtime耗时，e2eTime来自bmrt_test的calculate time\n",
    "e2eTime = 38300. # us\n",
    "# 模型的quantize_type影响该模型的PeakTops的选择\n",
    "quantize_type = 'int8'\n",
    "# 模型输入输出数据量\n",
    "ModelInputBytes = 16*3*224*224*4\n",
    "ModelOutputBytes = 16*1000*4\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "PeakTops = {\n",
    "  'int8': 32768 * tpu_freq,\n",
    "  'f16': 16384 * tpu_freq,\n",
    "  'bf16': 16384 * tpu_freq,\n",
    "  'f32': 2048 * tpu_freq,\n",
    "}\n",
    "ModelPeakTops = PeakTops[quantize_type]\n",
    "\n",
    "layer_df = pd.read_csv(profile_dir + 'layer.csv')\n",
    "summary_df = pd.read_csv(profile_dir + 'summary.csv')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Columns含义\n",
    "1. inputBytes：输入字节数，不包含weight\n",
    "2. outputBytes：输出字节数\n",
    "3. weightBytes：权重系数字节数\n",
    "4. s2lBytes：Globalmem -> Localmem字节数\n",
    "5. l2sBytes：Localmem -> Globalmem字节数\n",
    "6. s2sBytes：Globalmem -> Globalmem字节数\n",
    "7. gdmaCycles：GDMA运行的实际真实cycle数\n",
    "8. gdmaTime：GDMA运行的实际真实时间，即gdmaCycles * GDMAPeriod\n",
    "9. gdmaTimeRatio：GDMA耗时占GDMA总耗时的比例\n",
    "10. gdmaPTheoTime：根据profile中数据得出的GDMA理论耗时\n",
    "11. ddrRate：DDR效率，即gdmaPTheoTime / gdmaTime\n",
    "12. LoadAvgBandwidth：GDMA平均加载带宽（S2L）\n",
    "13. StoreAvgBandwidth：GDMA平均保存带宽（L2S）\n",
    "14. AlgOps：有效的算法计算量，**一般由于实现方案会和真实的该模型或Layer本身的计算量有一定的差异，模型本身的计算量请参考mlir中的数据**\n",
    "15. uArchOps：TPU微架构的计算量，相当于微架构利用率打满情况下的AlgOps\n",
    "16. uArchCModelCycles：cmodel仿真的cycle数，**如果与tiuCycles相差过大，说明有问题，需要校准cmodel中的仿真计算**\n",
    "17. uArchCModelCycleRatio：cmodel仿真中，该部分cycle占比\n",
    "18. tiuCycles：TIU运行时的实际真实cycle数\n",
    "19. tiuTime：TIU运行时的实际真实耗时，即tiuCycles * TIUPeriod\n",
    "20. tiuTimeRatio：该部分TIU耗时占TIU总耗时的占比\n",
    "21. tiuPTheoTime：根据profile中的数据得出的TIU理论耗时\n",
    "22. uArchRate：TPU微架构利用率，即AlgOps / uArchOps\n",
    "23. totalTime：该部分实际真实的总耗时，由于GDMA和TIU的并行，一般totalTime <= tiuTime + GDMATime\n",
    "24. PeakTops：该部分的峰值算力，由于使用的指令不同，不同算子对应的峰值算力也会有一定区别\n",
    "25. ActualTops：等效算力或实际使用到的算力，即AlgOps / totalTime\n",
    "26. Parallelism：该部分整体并行程度，可看出不并行时耗时的增加量，计算方式(tiuTime + gdmaTime) / totalTime\n",
    "27. Concurrency：该部分TIU与GDMA的并行程度，最大为100%，计算方式为(tiuTime + gdmaTime - totalTime) / min(tiuTime, gdmaTime)，"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>LayerID</th>\n",
       "      <th>Type</th>\n",
       "      <th>TPU/CPU</th>\n",
       "      <th>DataType</th>\n",
       "      <th>Function</th>\n",
       "      <th>in</th>\n",
       "      <th>ic</th>\n",
       "      <th>ih</th>\n",
       "      <th>iw</th>\n",
       "      <th>on</th>\n",
       "      <th>oc</th>\n",
       "      <th>oh</th>\n",
       "      <th>ow</th>\n",
       "      <th>kh</th>\n",
       "      <th>kw</th>\n",
       "      <th>KStrideH</th>\n",
       "      <th>KStrideW</th>\n",
       "      <th>Padding</th>\n",
       "      <th>Other info</th>\n",
       "      <th>inputBytes</th>\n",
       "      <th>outputBytes</th>\n",
       "      <th>weightBytes</th>\n",
       "      <th>s2lBytes</th>\n",
       "      <th>l2sBytes</th>\n",
       "      <th>s2sBytes</th>\n",
       "      <th>gdmaCycles</th>\n",
       "      <th>gdmaTime(us)</th>\n",
       "      <th>gdmaTimeRatio</th>\n",
       "      <th>gdmaPTheoTime(us)</th>\n",
       "      <th>ddrRate</th>\n",
       "      <th>LoadAvgBandwidth(GiB/s)</th>\n",
       "      <th>StoreAvgBandwidth(GiB/s)</th>\n",
       "      <th>AlgOps</th>\n",
       "      <th>uArchOps</th>\n",
       "      <th>uArchCModelCycles</th>\n",
       "      <th>uArchCModelCycleRatio</th>\n",
       "      <th>tiuCycles</th>\n",
       "      <th>tiuTime(us)</th>\n",
       "      <th>tiuTimeRatio</th>\n",
       "      <th>tiuPTheoTime(us)</th>\n",
       "      <th>uArchRate</th>\n",
       "      <th>totalTime(us)</th>\n",
       "      <th>PeakTops</th>\n",
       "      <th>ActualTops</th>\n",
       "      <th>Parallelism</th>\n",
       "      <th>Concurrency</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>267</td>\n",
       "      <td>local</td>\n",
       "      <td>TPU</td>\n",
       "      <td>{'FP32'}</td>\n",
       "      <td>Softmax</td>\n",
       "      <td>16</td>\n",
       "      <td>197</td>\n",
       "      <td>12</td>\n",
       "      <td>197</td>\n",
       "      <td>16</td>\n",
       "      <td>197</td>\n",
       "      <td>12</td>\n",
       "      <td>197</td>\n",
       "      <td>1</td>\n",
       "      <td>197</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>[0, 0, 0, 0]</td>\n",
       "      <td>ins=[tensor_id=266 [16x197x12x197] FP32  nslice=1 hslice=12],outs=[tensor_id=267 [16x197x12x197] FP32  nslice=1 hslice=12]</td>\n",
       "      <td>29805312</td>\n",
       "      <td>29805312</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000%</td>\n",
       "      <td>0.0</td>\n",
       "      <td>--</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>246017536</td>\n",
       "      <td>326713344</td>\n",
       "      <td>541392</td>\n",
       "      <td>1.717%</td>\n",
       "      <td>564736</td>\n",
       "      <td>564.736</td>\n",
       "      <td>1.624%</td>\n",
       "      <td>240.2515</td>\n",
       "      <td>75.301%</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>337</td>\n",
       "      <td>local</td>\n",
       "      <td>TPU</td>\n",
       "      <td>{'FP32'}</td>\n",
       "      <td>Softmax</td>\n",
       "      <td>16</td>\n",
       "      <td>197</td>\n",
       "      <td>12</td>\n",
       "      <td>197</td>\n",
       "      <td>16</td>\n",
       "      <td>197</td>\n",
       "      <td>12</td>\n",
       "      <td>197</td>\n",
       "      <td>1</td>\n",
       "      <td>197</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>[0, 0, 0, 0]</td>\n",
       "      <td>ins=[tensor_id=336 [16x197x12x197] FP32  nslice=1 hslice=12],outs=[tensor_id=337 [16x197x12x197] FP32  nslice=1 hslice=12]</td>\n",
       "      <td>29805312</td>\n",
       "      <td>29805312</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000%</td>\n",
       "      <td>0.0</td>\n",
       "      <td>--</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>246017536</td>\n",
       "      <td>326713344</td>\n",
       "      <td>541392</td>\n",
       "      <td>1.717%</td>\n",
       "      <td>564736</td>\n",
       "      <td>564.736</td>\n",
       "      <td>1.624%</td>\n",
       "      <td>240.2515</td>\n",
       "      <td>75.301%</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>407</td>\n",
       "      <td>local</td>\n",
       "      <td>TPU</td>\n",
       "      <td>{'FP32'}</td>\n",
       "      <td>Softmax</td>\n",
       "      <td>16</td>\n",
       "      <td>197</td>\n",
       "      <td>12</td>\n",
       "      <td>197</td>\n",
       "      <td>16</td>\n",
       "      <td>197</td>\n",
       "      <td>12</td>\n",
       "      <td>197</td>\n",
       "      <td>1</td>\n",
       "      <td>197</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>[0, 0, 0, 0]</td>\n",
       "      <td>ins=[tensor_id=406 [16x197x12x197] FP32  nslice=1 hslice=12],outs=[tensor_id=407 [16x197x12x197] FP32  nslice=1 hslice=12]</td>\n",
       "      <td>29805312</td>\n",
       "      <td>29805312</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000%</td>\n",
       "      <td>0.0</td>\n",
       "      <td>--</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>246017536</td>\n",
       "      <td>326713344</td>\n",
       "      <td>541392</td>\n",
       "      <td>1.717%</td>\n",
       "      <td>564736</td>\n",
       "      <td>564.736</td>\n",
       "      <td>1.624%</td>\n",
       "      <td>240.2515</td>\n",
       "      <td>75.301%</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>617</td>\n",
       "      <td>local</td>\n",
       "      <td>TPU</td>\n",
       "      <td>{'FP32'}</td>\n",
       "      <td>Softmax</td>\n",
       "      <td>16</td>\n",
       "      <td>197</td>\n",
       "      <td>12</td>\n",
       "      <td>197</td>\n",
       "      <td>16</td>\n",
       "      <td>197</td>\n",
       "      <td>12</td>\n",
       "      <td>197</td>\n",
       "      <td>1</td>\n",
       "      <td>197</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>[0, 0, 0, 0]</td>\n",
       "      <td>ins=[tensor_id=616 [16x197x12x197] FP32  nslice=1 hslice=12],outs=[tensor_id=617 [16x197x12x197] FP32  nslice=1 hslice=12]</td>\n",
       "      <td>29805312</td>\n",
       "      <td>29805312</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000%</td>\n",
       "      <td>0.0</td>\n",
       "      <td>--</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>246017536</td>\n",
       "      <td>326713344</td>\n",
       "      <td>541392</td>\n",
       "      <td>1.717%</td>\n",
       "      <td>564736</td>\n",
       "      <td>564.736</td>\n",
       "      <td>1.624%</td>\n",
       "      <td>240.2515</td>\n",
       "      <td>75.301%</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>757</td>\n",
       "      <td>local</td>\n",
       "      <td>TPU</td>\n",
       "      <td>{'FP32'}</td>\n",
       "      <td>Softmax</td>\n",
       "      <td>16</td>\n",
       "      <td>197</td>\n",
       "      <td>12</td>\n",
       "      <td>197</td>\n",
       "      <td>16</td>\n",
       "      <td>197</td>\n",
       "      <td>12</td>\n",
       "      <td>197</td>\n",
       "      <td>1</td>\n",
       "      <td>197</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>[0, 0, 0, 0]</td>\n",
       "      <td>ins=[tensor_id=756 [16x197x12x197] FP32  nslice=1 hslice=12],outs=[tensor_id=757 [16x197x12x197] FP32  nslice=1 hslice=12]</td>\n",
       "      <td>29805312</td>\n",
       "      <td>29805312</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000%</td>\n",
       "      <td>0.0</td>\n",
       "      <td>--</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>246017536</td>\n",
       "      <td>326713344</td>\n",
       "      <td>541392</td>\n",
       "      <td>1.717%</td>\n",
       "      <td>564736</td>\n",
       "      <td>564.736</td>\n",
       "      <td>1.624%</td>\n",
       "      <td>240.2515</td>\n",
       "      <td>75.301%</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   LayerID   Type TPU/CPU  DataType Function  in   ic  ih   iw  on   oc  oh   ow  kh   kw  KStrideH  KStrideW       Padding                                                                                                                  Other info  inputBytes  outputBytes  weightBytes  s2lBytes  l2sBytes  s2sBytes  gdmaCycles  gdmaTime(us) gdmaTimeRatio  gdmaPTheoTime(us) ddrRate  LoadAvgBandwidth(GiB/s)  StoreAvgBandwidth(GiB/s)     AlgOps   uArchOps  uArchCModelCycles uArchCModelCycleRatio  tiuCycles  tiuTime(us) tiuTimeRatio  tiuPTheoTime(us) uArchRate  totalTime(us)  PeakTops  ActualTops Parallelism Concurrency\n",
       "0      267  local     TPU  {'FP32'}  Softmax  16  197  12  197  16  197  12  197   1  197         1         1  [0, 0, 0, 0]  ins=[tensor_id=266 [16x197x12x197] FP32  nslice=1 hslice=12],outs=[tensor_id=267 [16x197x12x197] FP32  nslice=1 hslice=12]    29805312     29805312            0         0         0         0           0           0.0        0.000%                0.0      --                      0.0                       0.0  246017536  326713344             541392                1.717%     564736      564.736       1.624%          240.2515   75.301%            NaN       1.0         NaN         NaN         NaN\n",
       "1      337  local     TPU  {'FP32'}  Softmax  16  197  12  197  16  197  12  197   1  197         1         1  [0, 0, 0, 0]  ins=[tensor_id=336 [16x197x12x197] FP32  nslice=1 hslice=12],outs=[tensor_id=337 [16x197x12x197] FP32  nslice=1 hslice=12]    29805312     29805312            0         0         0         0           0           0.0        0.000%                0.0      --                      0.0                       0.0  246017536  326713344             541392                1.717%     564736      564.736       1.624%          240.2515   75.301%            NaN       1.0         NaN         NaN         NaN\n",
       "2      407  local     TPU  {'FP32'}  Softmax  16  197  12  197  16  197  12  197   1  197         1         1  [0, 0, 0, 0]  ins=[tensor_id=406 [16x197x12x197] FP32  nslice=1 hslice=12],outs=[tensor_id=407 [16x197x12x197] FP32  nslice=1 hslice=12]    29805312     29805312            0         0         0         0           0           0.0        0.000%                0.0      --                      0.0                       0.0  246017536  326713344             541392                1.717%     564736      564.736       1.624%          240.2515   75.301%            NaN       1.0         NaN         NaN         NaN\n",
       "3      617  local     TPU  {'FP32'}  Softmax  16  197  12  197  16  197  12  197   1  197         1         1  [0, 0, 0, 0]  ins=[tensor_id=616 [16x197x12x197] FP32  nslice=1 hslice=12],outs=[tensor_id=617 [16x197x12x197] FP32  nslice=1 hslice=12]    29805312     29805312            0         0         0         0           0           0.0        0.000%                0.0      --                      0.0                       0.0  246017536  326713344             541392                1.717%     564736      564.736       1.624%          240.2515   75.301%            NaN       1.0         NaN         NaN         NaN\n",
       "4      757  local     TPU  {'FP32'}  Softmax  16  197  12  197  16  197  12  197   1  197         1         1  [0, 0, 0, 0]  ins=[tensor_id=756 [16x197x12x197] FP32  nslice=1 hslice=12],outs=[tensor_id=757 [16x197x12x197] FP32  nslice=1 hslice=12]    29805312     29805312            0         0         0         0           0           0.0        0.000%                0.0      --                      0.0                       0.0  246017536  326713344             541392                1.717%     564736      564.736       1.624%          240.2515   75.301%            NaN       1.0         NaN         NaN         NaN"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "layer_df.head()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Function</th>\n",
       "      <th>weightBytes</th>\n",
       "      <th>s2lBytes</th>\n",
       "      <th>l2sBytes</th>\n",
       "      <th>s2sBytes</th>\n",
       "      <th>gdmaCycles</th>\n",
       "      <th>gdmaTime(us)</th>\n",
       "      <th>gdmaTimeRatio</th>\n",
       "      <th>gdmaPTheoTime(us)</th>\n",
       "      <th>ddrRate</th>\n",
       "      <th>AlgOps</th>\n",
       "      <th>AlgOpsRatio</th>\n",
       "      <th>uArchOps</th>\n",
       "      <th>uArchOpsRatio</th>\n",
       "      <th>tiuCycles</th>\n",
       "      <th>tiuTime(us)</th>\n",
       "      <th>tiuTimeRatio</th>\n",
       "      <th>tiuPTheoTime(us)</th>\n",
       "      <th>uArchRate</th>\n",
       "      <th>PeakTops</th>\n",
       "      <th>DataTypes</th>\n",
       "      <th>LayerTypes</th>\n",
       "      <th>totalTime(us)</th>\n",
       "      <th>Parallelism</th>\n",
       "      <th>Concurrency</th>\n",
       "      <th>1684x FPS or Token/s</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Matmul</td>\n",
       "      <td>86631328</td>\n",
       "      <td>3785632</td>\n",
       "      <td>2424448</td>\n",
       "      <td>0</td>\n",
       "      <td>129324</td>\n",
       "      <td>129.324</td>\n",
       "      <td>2.071%</td>\n",
       "      <td>112.103865</td>\n",
       "      <td>86.685%</td>\n",
       "      <td>563790963392</td>\n",
       "      <td>99.106%</td>\n",
       "      <td>622837575680</td>\n",
       "      <td>98.967%</td>\n",
       "      <td>23054565</td>\n",
       "      <td>23054.565</td>\n",
       "      <td>66.297%</td>\n",
       "      <td>17205.534771</td>\n",
       "      <td>90.520%</td>\n",
       "      <td>32.0000</td>\n",
       "      <td>UINT8,INT8</td>\n",
       "      <td>MatMul</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Softmax</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000%</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>--</td>\n",
       "      <td>2952210432</td>\n",
       "      <td>0.519%</td>\n",
       "      <td>3920560128</td>\n",
       "      <td>0.623%</td>\n",
       "      <td>6776384</td>\n",
       "      <td>6776.384</td>\n",
       "      <td>19.487%</td>\n",
       "      <td>2883.018000</td>\n",
       "      <td>75.301%</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>FP32</td>\n",
       "      <td>Softmax</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Cast</td>\n",
       "      <td>0</td>\n",
       "      <td>9649792</td>\n",
       "      <td>2472448</td>\n",
       "      <td>0</td>\n",
       "      <td>224854</td>\n",
       "      <td>224.854</td>\n",
       "      <td>3.601%</td>\n",
       "      <td>207.282346</td>\n",
       "      <td>92.185%</td>\n",
       "      <td>1075664256</td>\n",
       "      <td>0.189%</td>\n",
       "      <td>1333805056</td>\n",
       "      <td>0.212%</td>\n",
       "      <td>1364047</td>\n",
       "      <td>1364.047</td>\n",
       "      <td>3.923%</td>\n",
       "      <td>577.617281</td>\n",
       "      <td>80.646%</td>\n",
       "      <td>4.0000</td>\n",
       "      <td>FP32,FP16,INT8</td>\n",
       "      <td>Cast</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>LayerNorm</td>\n",
       "      <td>153600</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000%</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>--</td>\n",
       "      <td>628983224</td>\n",
       "      <td>0.111%</td>\n",
       "      <td>772317184</td>\n",
       "      <td>0.123%</td>\n",
       "      <td>1495611</td>\n",
       "      <td>1495.611</td>\n",
       "      <td>4.301%</td>\n",
       "      <td>307.120715</td>\n",
       "      <td>81.441%</td>\n",
       "      <td>2.0000</td>\n",
       "      <td>FP16</td>\n",
       "      <td>LayerNorm</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Eltwise</td>\n",
       "      <td>151296</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000%</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>--</td>\n",
       "      <td>210604032</td>\n",
       "      <td>0.037%</td>\n",
       "      <td>226197504</td>\n",
       "      <td>0.036%</td>\n",
       "      <td>110164</td>\n",
       "      <td>110.164</td>\n",
       "      <td>0.317%</td>\n",
       "      <td>51.417000</td>\n",
       "      <td>93.106%</td>\n",
       "      <td>4.0000</td>\n",
       "      <td>INT8</td>\n",
       "      <td>MulShift,Add</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Lut</td>\n",
       "      <td>3072</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000%</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>--</td>\n",
       "      <td>116998144</td>\n",
       "      <td>0.021%</td>\n",
       "      <td>118767616</td>\n",
       "      <td>0.019%</td>\n",
       "      <td>1845258</td>\n",
       "      <td>1845.258</td>\n",
       "      <td>5.306%</td>\n",
       "      <td>1828.096000</td>\n",
       "      <td>98.510%</td>\n",
       "      <td>0.0625</td>\n",
       "      <td>INT8</td>\n",
       "      <td>Lut</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Others</td>\n",
       "      <td>24576</td>\n",
       "      <td>7225344</td>\n",
       "      <td>7225344</td>\n",
       "      <td>12288</td>\n",
       "      <td>998330</td>\n",
       "      <td>998.330</td>\n",
       "      <td>15.987%</td>\n",
       "      <td>269.362812</td>\n",
       "      <td>26.981%</td>\n",
       "      <td>99213312</td>\n",
       "      <td>0.017%</td>\n",
       "      <td>127909888</td>\n",
       "      <td>0.020%</td>\n",
       "      <td>128499</td>\n",
       "      <td>128.499</td>\n",
       "      <td>0.370%</td>\n",
       "      <td>24.813000</td>\n",
       "      <td>77.565%</td>\n",
       "      <td>4.0000</td>\n",
       "      <td>FP16,INT8</td>\n",
       "      <td>Permute,Slice,Concat,Reshape</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Load</td>\n",
       "      <td>85598976</td>\n",
       "      <td>170312448</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3554002</td>\n",
       "      <td>3554.002</td>\n",
       "      <td>56.913%</td>\n",
       "      <td>2734.755648</td>\n",
       "      <td>76.949%</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000%</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000%</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000%</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>--</td>\n",
       "      <td>4.0000</td>\n",
       "      <td>FP32,FP16,INT32,INT8</td>\n",
       "      <td>Load</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Store</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>58097664</td>\n",
       "      <td>0</td>\n",
       "      <td>1338139</td>\n",
       "      <td>1338.139</td>\n",
       "      <td>21.429%</td>\n",
       "      <td>1229.719682</td>\n",
       "      <td>91.898%</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000%</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000%</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000%</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>--</td>\n",
       "      <td>4.0000</td>\n",
       "      <td>INT8</td>\n",
       "      <td>Store</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Overall</td>\n",
       "      <td>86963872</td>\n",
       "      <td>190973216</td>\n",
       "      <td>70219904</td>\n",
       "      <td>12288</td>\n",
       "      <td>6244649</td>\n",
       "      <td>6244.649</td>\n",
       "      <td>100%</td>\n",
       "      <td>4553.224353</td>\n",
       "      <td>72.914%</td>\n",
       "      <td>568874636792</td>\n",
       "      <td>100%</td>\n",
       "      <td>629337133056</td>\n",
       "      <td>100%</td>\n",
       "      <td>34774528</td>\n",
       "      <td>34774.528</td>\n",
       "      <td>100%</td>\n",
       "      <td>22877.616768</td>\n",
       "      <td>90.393%</td>\n",
       "      <td>32.0000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>37701.045</td>\n",
       "      <td>108.801%</td>\n",
       "      <td>53.136%</td>\n",
       "      <td>28.756681</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    Function  weightBytes   s2lBytes  l2sBytes  s2sBytes  gdmaCycles  gdmaTime(us) gdmaTimeRatio  gdmaPTheoTime(us)  ddrRate        AlgOps AlgOpsRatio      uArchOps uArchOpsRatio  tiuCycles  tiuTime(us) tiuTimeRatio  tiuPTheoTime(us) uArchRate  PeakTops             DataTypes                    LayerTypes  totalTime(us) Parallelism Concurrency  1684x FPS or Token/s\n",
       "0     Matmul     86631328    3785632   2424448         0      129324       129.324        2.071%         112.103865  86.685%  563790963392     99.106%  622837575680       98.967%   23054565    23054.565      66.297%      17205.534771   90.520%   32.0000            UINT8,INT8                        MatMul            NaN           0           0              0.000000\n",
       "1    Softmax            0          0         0         0           0         0.000        0.000%           0.000000       --    2952210432      0.519%    3920560128        0.623%    6776384     6776.384      19.487%       2883.018000   75.301%    1.0000                  FP32                       Softmax            NaN           0           0              0.000000\n",
       "2       Cast            0    9649792   2472448         0      224854       224.854        3.601%         207.282346  92.185%    1075664256      0.189%    1333805056        0.212%    1364047     1364.047       3.923%        577.617281   80.646%    4.0000        FP32,FP16,INT8                          Cast            NaN           0           0              0.000000\n",
       "3  LayerNorm       153600          0         0         0           0         0.000        0.000%           0.000000       --     628983224      0.111%     772317184        0.123%    1495611     1495.611       4.301%        307.120715   81.441%    2.0000                  FP16                     LayerNorm            NaN           0           0              0.000000\n",
       "4    Eltwise       151296          0         0         0           0         0.000        0.000%           0.000000       --     210604032      0.037%     226197504        0.036%     110164      110.164       0.317%         51.417000   93.106%    4.0000                  INT8                  MulShift,Add            NaN           0           0              0.000000\n",
       "5        Lut         3072          0         0         0           0         0.000        0.000%           0.000000       --     116998144      0.021%     118767616        0.019%    1845258     1845.258       5.306%       1828.096000   98.510%    0.0625                  INT8                           Lut            NaN           0           0              0.000000\n",
       "6     Others        24576    7225344   7225344     12288      998330       998.330       15.987%         269.362812  26.981%      99213312      0.017%     127909888        0.020%     128499      128.499       0.370%         24.813000   77.565%    4.0000             FP16,INT8  Permute,Slice,Concat,Reshape            NaN           0           0              0.000000\n",
       "7       Load     85598976  170312448         0         0     3554002      3554.002       56.913%        2734.755648  76.949%             0      0.000%             0        0.000%          0        0.000       0.000%          0.000000        --    4.0000  FP32,FP16,INT32,INT8                          Load            NaN           0           0              0.000000\n",
       "8      Store            0          0  58097664         0     1338139      1338.139       21.429%        1229.719682  91.898%             0      0.000%             0        0.000%          0        0.000       0.000%          0.000000        --    4.0000                  INT8                         Store            NaN           0           0              0.000000\n",
       "9    Overall     86963872  190973216  70219904     12288     6244649      6244.649          100%        4553.224353  72.914%  568874636792        100%  629337133056          100%   34774528    34774.528         100%      22877.616768   90.393%   32.0000                   NaN                           NaN      37701.045    108.801%     53.136%             28.756681"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "summary_df\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Function\n",
       "Add           25\n",
       "Cast          78\n",
       "Concat         1\n",
       "LayerNorm     25\n",
       "Load         243\n",
       "Lut           12\n",
       "MatMul        98\n",
       "MulShift      12\n",
       "Permute        1\n",
       "Reshape       37\n",
       "Slice          1\n",
       "Softmax       12\n",
       "Store         24\n",
       "Name: Function, dtype: int64"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "layer_df.groupby('Function')['Function'].count()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1. ddrDataSize优化\n",
    "\n",
    "ddrDataSize = s2lBytes + l2sBytes + s2sBytes\n",
    "\n",
    "ddrMinDataSize = ModelInputBytes + ModelOutputBytes + ModelWeightBytes - UnusedWeightBytes\n",
    "\n",
    "其中，**ModelInputBytes和ModelOutputBytes暂时无法从profile中获取到，需要自行计算填入**，ModelWeightBytes = sum(weightBytes), UnusedWeightBytes = GatherOpWeightBytes - GatherOpOutputBytes\n",
    "\n",
    "ddrMinDataSize是不考虑LocalMem等各种因素情况下运行该模型的最小数据搬运量，ddrDataSize是实际的搬运量，应当尽可能让ddrDataSize接近ddrMinDataSize，可从如下几个方面着手：\n",
    "1. 图优化：消除冗余算子，transformer的图优化就比较典型，消除了大量的permute，从而减少了搬运，这一步可通过Netron查看tpu_opt.onnx来进行\n",
    "2. 遍历每一个global layer，弄清楚作为global layer的原因，然后尽可能减少global layer的存在；无法避免只能作为global layer的，尽可能减少重复搬运\n",
    "3. 关注Concat、Slice、Permute、Pad等几乎没有计算，只有数据搬运的算子，可能存在某些特殊的优化方法，如inplace方法或在Load、Store时完成相关操作等"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ddrDataSize = 261205408 Bytes, ddrMinDataSize = 182260640 Bytes\n"
     ]
    }
   ],
   "source": [
    "column_names = [\n",
    "    'LayerID', 'Type', 'DataType', 'Function',\n",
    "    'inputBytes', 'outputBytes', 'weightBytes',\n",
    "    's2lBytes', 'l2sBytes', 's2sBytes'\n",
    "]\n",
    "\n",
    "\n",
    "gather_df = layer_df.loc[layer_df['Function'] == 'Gather']\n",
    "UnusedWeightBytes = gather_df['weightBytes'].sum() - gather_df['outputBytes'].sum()\n",
    "ModelWeightBytes = layer_df['weightBytes'].sum()\n",
    "ddrMinDataSize = ModelInputBytes + ModelOutputBytes + ModelWeightBytes - UnusedWeightBytes\n",
    "\n",
    "ddrDataSize = layer_df[['s2lBytes', 'l2sBytes', 's2sBytes']].sum().sum()\n",
    "print(f\"ddrDataSize = {ddrDataSize} Bytes, ddrMinDataSize = {ddrMinDataSize} Bytes\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2. uArchRate优化"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "从下表可以看出各类型Layer的uArchRate，得到一个基本信息，可以看出耗时占比最大的是MatMul和Softmax，它们的uArchRate依然有提升空间，其他几个虽然有提升空间，但相对收益要低一些，可以放在后面在考虑"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>uArchRate</th>\n",
       "      <th>Function</th>\n",
       "      <th>AlgOps</th>\n",
       "      <th>AlgOpsRatio</th>\n",
       "      <th>uArchOps</th>\n",
       "      <th>uArchOpsRatio</th>\n",
       "      <th>tiuTime(us)</th>\n",
       "      <th>tiuTimeRatio</th>\n",
       "      <th>tiuPTheoTime(us)</th>\n",
       "      <th>PeakTops</th>\n",
       "      <th>DataTypes</th>\n",
       "      <th>LayerTypes</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>90.393%</td>\n",
       "      <td>Overall</td>\n",
       "      <td>568874636792</td>\n",
       "      <td>100%</td>\n",
       "      <td>629337133056</td>\n",
       "      <td>100%</td>\n",
       "      <td>34774.528</td>\n",
       "      <td>100%</td>\n",
       "      <td>22877.616768</td>\n",
       "      <td>32.0000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>90.520%</td>\n",
       "      <td>Matmul</td>\n",
       "      <td>563790963392</td>\n",
       "      <td>99.106%</td>\n",
       "      <td>622837575680</td>\n",
       "      <td>98.967%</td>\n",
       "      <td>23054.565</td>\n",
       "      <td>66.297%</td>\n",
       "      <td>17205.534771</td>\n",
       "      <td>32.0000</td>\n",
       "      <td>UINT8,INT8</td>\n",
       "      <td>MatMul</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>75.301%</td>\n",
       "      <td>Softmax</td>\n",
       "      <td>2952210432</td>\n",
       "      <td>0.519%</td>\n",
       "      <td>3920560128</td>\n",
       "      <td>0.623%</td>\n",
       "      <td>6776.384</td>\n",
       "      <td>19.487%</td>\n",
       "      <td>2883.018000</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>FP32</td>\n",
       "      <td>Softmax</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>98.510%</td>\n",
       "      <td>Lut</td>\n",
       "      <td>116998144</td>\n",
       "      <td>0.021%</td>\n",
       "      <td>118767616</td>\n",
       "      <td>0.019%</td>\n",
       "      <td>1845.258</td>\n",
       "      <td>5.306%</td>\n",
       "      <td>1828.096000</td>\n",
       "      <td>0.0625</td>\n",
       "      <td>INT8</td>\n",
       "      <td>Lut</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>81.441%</td>\n",
       "      <td>LayerNorm</td>\n",
       "      <td>628983224</td>\n",
       "      <td>0.111%</td>\n",
       "      <td>772317184</td>\n",
       "      <td>0.123%</td>\n",
       "      <td>1495.611</td>\n",
       "      <td>4.301%</td>\n",
       "      <td>307.120715</td>\n",
       "      <td>2.0000</td>\n",
       "      <td>FP16</td>\n",
       "      <td>LayerNorm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>80.646%</td>\n",
       "      <td>Cast</td>\n",
       "      <td>1075664256</td>\n",
       "      <td>0.189%</td>\n",
       "      <td>1333805056</td>\n",
       "      <td>0.212%</td>\n",
       "      <td>1364.047</td>\n",
       "      <td>3.923%</td>\n",
       "      <td>577.617281</td>\n",
       "      <td>4.0000</td>\n",
       "      <td>FP32,FP16,INT8</td>\n",
       "      <td>Cast</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>77.565%</td>\n",
       "      <td>Others</td>\n",
       "      <td>99213312</td>\n",
       "      <td>0.017%</td>\n",
       "      <td>127909888</td>\n",
       "      <td>0.020%</td>\n",
       "      <td>128.499</td>\n",
       "      <td>0.370%</td>\n",
       "      <td>24.813000</td>\n",
       "      <td>4.0000</td>\n",
       "      <td>FP16,INT8</td>\n",
       "      <td>Permute,Slice,Concat,Reshape</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>93.106%</td>\n",
       "      <td>Eltwise</td>\n",
       "      <td>210604032</td>\n",
       "      <td>0.037%</td>\n",
       "      <td>226197504</td>\n",
       "      <td>0.036%</td>\n",
       "      <td>110.164</td>\n",
       "      <td>0.317%</td>\n",
       "      <td>51.417000</td>\n",
       "      <td>4.0000</td>\n",
       "      <td>INT8</td>\n",
       "      <td>MulShift,Add</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>--</td>\n",
       "      <td>Load</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000%</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000%</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000%</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>4.0000</td>\n",
       "      <td>FP32,FP16,INT32,INT8</td>\n",
       "      <td>Load</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>--</td>\n",
       "      <td>Store</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000%</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000%</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000%</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>4.0000</td>\n",
       "      <td>INT8</td>\n",
       "      <td>Store</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  uArchRate   Function        AlgOps AlgOpsRatio      uArchOps uArchOpsRatio  tiuTime(us) tiuTimeRatio  tiuPTheoTime(us)  PeakTops             DataTypes                    LayerTypes\n",
       "9   90.393%    Overall  568874636792        100%  629337133056          100%    34774.528         100%      22877.616768   32.0000                   NaN                           NaN\n",
       "0   90.520%     Matmul  563790963392     99.106%  622837575680       98.967%    23054.565      66.297%      17205.534771   32.0000            UINT8,INT8                        MatMul\n",
       "1   75.301%    Softmax    2952210432      0.519%    3920560128        0.623%     6776.384      19.487%       2883.018000    1.0000                  FP32                       Softmax\n",
       "5   98.510%        Lut     116998144      0.021%     118767616        0.019%     1845.258       5.306%       1828.096000    0.0625                  INT8                           Lut\n",
       "3   81.441%  LayerNorm     628983224      0.111%     772317184        0.123%     1495.611       4.301%        307.120715    2.0000                  FP16                     LayerNorm\n",
       "2   80.646%       Cast    1075664256      0.189%    1333805056        0.212%     1364.047       3.923%        577.617281    4.0000        FP32,FP16,INT8                          Cast\n",
       "6   77.565%     Others      99213312      0.017%     127909888        0.020%      128.499       0.370%         24.813000    4.0000             FP16,INT8  Permute,Slice,Concat,Reshape\n",
       "4   93.106%    Eltwise     210604032      0.037%     226197504        0.036%      110.164       0.317%         51.417000    4.0000                  INT8                  MulShift,Add\n",
       "7        --       Load             0      0.000%             0        0.000%        0.000       0.000%          0.000000    4.0000  FP32,FP16,INT32,INT8                          Load\n",
       "8        --      Store             0      0.000%             0        0.000%        0.000       0.000%          0.000000    4.0000                  INT8                         Store"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "column_names = [\n",
    "    'uArchRate', 'Function', 'AlgOps', 'AlgOpsRatio', 'uArchOps', 'uArchOpsRatio',\n",
    "    'tiuTime(us)', 'tiuTimeRatio', 'tiuPTheoTime(us)',\n",
    "    'PeakTops', 'DataTypes', 'LayerTypes'\n",
    "]\n",
    "arch_summary_df = summary_df[column_names]\n",
    "arch_summary_df.sort_values(by='tiuTime(us)', axis=0, ascending=False)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## MatMul\n",
    "### step1\n",
    "可以看到MatMul的uArchRate的主要有7种情况：\n",
    "1. 98.5%对应的tiuTime为12.951ms，由于本身uArchRate接近100%，优化空间很小；\n",
    "2. 2个接近76%的利用率总计耗时8.865ms，有较大的优化空间；\n",
    "3. 剩余4中情况本身耗时很短，优化提升较小，可放在最后再考虑优化。\n",
    "\n",
    "**注：优化到最后，不论哪一种情况都应该是进行到无法再优化的程度，并列出原因，如果因为时间问题来不及做，可简略备注一下。**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "uArchRate\n",
       "100.000%      133.630\n",
       "24.999%         3.483\n",
       "59.387%       917.006\n",
       "59.388%       183.513\n",
       "76.683%      2896.128\n",
       "76.957%      5969.136\n",
       "98.500%     12951.669\n",
       "Name: tiuTime(us), dtype: float64"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "layer_df.loc[layer_df['Function'] == 'MatMul'].groupby('uArchRate')['tiuTime(us)'].sum()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### step2\n",
    "重点关注2个76%的MatMul的情况，如果下图查看不方便，可以导出到表格中查看，这部分可以对照final.mlir查看为什么这些MatMul的利用率低，是LayerGroup问题，还是后端实现问题。\n",
    "\n",
    "这里的利用率低主要都是因为左矩阵的C=197，导致无法充分利用lane，需要采取策略，尽可能打满lane"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_layers(layer_df, layer_type:str):\n",
    "    column_names = [\n",
    "        'uArchRate', 'LayerID', 'Type', 'DataType', 'Function', 'Other info',\n",
    "        'AlgOps', 'uArchOps', 'tiuTime(us)', 'tiuTimeRatio', 'tiuPTheoTime(us)',\n",
    "        'totalTime(us)', 'PeakTops', 'ActualTops', 'Concurrency'\n",
    "    ]\n",
    "    return layer_df[column_names].loc[layer_df['Function'] == layer_type].sort_values(by='uArchRate', axis=0)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>uArchRate</th>\n",
       "      <th>LayerID</th>\n",
       "      <th>Type</th>\n",
       "      <th>DataType</th>\n",
       "      <th>Function</th>\n",
       "      <th>Other info</th>\n",
       "      <th>AlgOps</th>\n",
       "      <th>uArchOps</th>\n",
       "      <th>tiuTime(us)</th>\n",
       "      <th>tiuTimeRatio</th>\n",
       "      <th>tiuPTheoTime(us)</th>\n",
       "      <th>totalTime(us)</th>\n",
       "      <th>PeakTops</th>\n",
       "      <th>ActualTops</th>\n",
       "      <th>Concurrency</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>97</th>\n",
       "      <td>100.000%</td>\n",
       "      <td>19</td>\n",
       "      <td>global</td>\n",
       "      <td>{'INT8'}</td>\n",
       "      <td>MatMul</td>\n",
       "      <td>ins=[tensor_id=16 [16x196x768] INT8  ,tensor_id=17 [1x768x768] INT8 CONST ,tensor_id=18 [1x1x768] INT32 CONST ],outs=[tensor_id=19 [16x196x768] INT8  ]</td>\n",
       "      <td>3709009920</td>\n",
       "      <td>3709009920</td>\n",
       "      <td>133.630</td>\n",
       "      <td>0.384%</td>\n",
       "      <td>113.190000</td>\n",
       "      <td>172.73</td>\n",
       "      <td>32.0</td>\n",
       "      <td>21.472876</td>\n",
       "      <td>62.880%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>250</th>\n",
       "      <td>24.999%</td>\n",
       "      <td>878</td>\n",
       "      <td>global</td>\n",
       "      <td>{'INT8'}</td>\n",
       "      <td>MatMul</td>\n",
       "      <td>ins=[tensor_id=875 [16x768] INT8  ,tensor_id=876 [768x1000] INT8 CONST ,tensor_id=877 [1x1000] INT32 CONST ],outs=[tensor_id=878 [16x1000] INT8  ]</td>\n",
       "      <td>24640000</td>\n",
       "      <td>98562048</td>\n",
       "      <td>3.483</td>\n",
       "      <td>0.010%</td>\n",
       "      <td>0.751953</td>\n",
       "      <td>27.49</td>\n",
       "      <td>32.0</td>\n",
       "      <td>0.896326</td>\n",
       "      <td>-0.459%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>109</th>\n",
       "      <td>59.387%</td>\n",
       "      <td>129</td>\n",
       "      <td>local</td>\n",
       "      <td>{'UINT8'}</td>\n",
       "      <td>MatMul</td>\n",
       "      <td>ins=[tensor_id=128 [16x197x12x197] UINT8  nslice=1 hslice=12,tensor_id=123 [16x197x12x64] INT8  nslice=1 hslice=12],outs=[tensor_id=129 [16x197x12x64] INT8  nslice=1 hslice=12]</td>\n",
       "      <td>973236736</td>\n",
       "      <td>1638809600</td>\n",
       "      <td>91.685</td>\n",
       "      <td>0.264%</td>\n",
       "      <td>29.700828</td>\n",
       "      <td>NaN</td>\n",
       "      <td>32.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>110</th>\n",
       "      <td>59.387%</td>\n",
       "      <td>479</td>\n",
       "      <td>local</td>\n",
       "      <td>{'UINT8'}</td>\n",
       "      <td>MatMul</td>\n",
       "      <td>ins=[tensor_id=478 [16x197x12x197] UINT8  nslice=1 hslice=12,tensor_id=473 [16x197x12x64] INT8  nslice=1 hslice=12],outs=[tensor_id=479 [16x197x12x64] INT8  nslice=1 hslice=12]</td>\n",
       "      <td>973236736</td>\n",
       "      <td>1638809600</td>\n",
       "      <td>91.685</td>\n",
       "      <td>0.264%</td>\n",
       "      <td>29.700828</td>\n",
       "      <td>NaN</td>\n",
       "      <td>32.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>111</th>\n",
       "      <td>59.387%</td>\n",
       "      <td>759</td>\n",
       "      <td>local</td>\n",
       "      <td>{'UINT8'}</td>\n",
       "      <td>MatMul</td>\n",
       "      <td>ins=[tensor_id=758 [16x197x12x197] UINT8  nslice=1 hslice=12,tensor_id=753 [16x197x12x64] INT8  nslice=1 hslice=12],outs=[tensor_id=759 [16x197x12x64] INT8  nslice=1 hslice=12]</td>\n",
       "      <td>973261952</td>\n",
       "      <td>1638842368</td>\n",
       "      <td>91.711</td>\n",
       "      <td>0.264%</td>\n",
       "      <td>29.701598</td>\n",
       "      <td>NaN</td>\n",
       "      <td>32.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    uArchRate  LayerID    Type   DataType Function                                                                                                                                                                        Other info      AlgOps    uArchOps  tiuTime(us) tiuTimeRatio  tiuPTheoTime(us)  totalTime(us)  PeakTops  ActualTops Concurrency\n",
       "97   100.000%       19  global   {'INT8'}   MatMul                           ins=[tensor_id=16 [16x196x768] INT8  ,tensor_id=17 [1x768x768] INT8 CONST ,tensor_id=18 [1x1x768] INT32 CONST ],outs=[tensor_id=19 [16x196x768] INT8  ]  3709009920  3709009920      133.630       0.384%        113.190000         172.73      32.0   21.472876     62.880%\n",
       "250   24.999%      878  global   {'INT8'}   MatMul                                ins=[tensor_id=875 [16x768] INT8  ,tensor_id=876 [768x1000] INT8 CONST ,tensor_id=877 [1x1000] INT32 CONST ],outs=[tensor_id=878 [16x1000] INT8  ]    24640000    98562048        3.483       0.010%          0.751953          27.49      32.0    0.896326     -0.459%\n",
       "109   59.387%      129   local  {'UINT8'}   MatMul  ins=[tensor_id=128 [16x197x12x197] UINT8  nslice=1 hslice=12,tensor_id=123 [16x197x12x64] INT8  nslice=1 hslice=12],outs=[tensor_id=129 [16x197x12x64] INT8  nslice=1 hslice=12]   973236736  1638809600       91.685       0.264%         29.700828            NaN      32.0         NaN         NaN\n",
       "110   59.387%      479   local  {'UINT8'}   MatMul  ins=[tensor_id=478 [16x197x12x197] UINT8  nslice=1 hslice=12,tensor_id=473 [16x197x12x64] INT8  nslice=1 hslice=12],outs=[tensor_id=479 [16x197x12x64] INT8  nslice=1 hslice=12]   973236736  1638809600       91.685       0.264%         29.700828            NaN      32.0         NaN         NaN\n",
       "111   59.387%      759   local  {'UINT8'}   MatMul  ins=[tensor_id=758 [16x197x12x197] UINT8  nslice=1 hslice=12,tensor_id=753 [16x197x12x64] INT8  nslice=1 hslice=12],outs=[tensor_id=759 [16x197x12x64] INT8  nslice=1 hslice=12]   973261952  1638842368       91.711       0.264%         29.701598            NaN      32.0         NaN         NaN"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "matmul = get_layers(layer_df, 'MatMul')\n",
    "matmul.to_csv(profile_dir + 'matmu.csv')\n",
    "matmul.head()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Softmax\n",
    "### step1\n",
    "可以看到，Softmax只有1种情况，因此只需要考虑优化这种情况即可"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "uArchRate\n",
       "75.301%    6776.384\n",
       "Name: tiuTime(us), dtype: float64"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "layer_df.loc[layer_df['Function'] == 'Softmax'].groupby('uArchRate')['tiuTime(us)'].sum()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### step2\n",
    "同样，Softmax利用率低的原因是因为C=197，无法充分利用lane"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>uArchRate</th>\n",
       "      <th>LayerID</th>\n",
       "      <th>Type</th>\n",
       "      <th>DataType</th>\n",
       "      <th>Function</th>\n",
       "      <th>Other info</th>\n",
       "      <th>AlgOps</th>\n",
       "      <th>uArchOps</th>\n",
       "      <th>tiuTime(us)</th>\n",
       "      <th>tiuTimeRatio</th>\n",
       "      <th>tiuPTheoTime(us)</th>\n",
       "      <th>totalTime(us)</th>\n",
       "      <th>PeakTops</th>\n",
       "      <th>ActualTops</th>\n",
       "      <th>Concurrency</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>75.301%</td>\n",
       "      <td>267</td>\n",
       "      <td>local</td>\n",
       "      <td>{'FP32'}</td>\n",
       "      <td>Softmax</td>\n",
       "      <td>ins=[tensor_id=266 [16x197x12x197] FP32  nslice=1 hslice=12],outs=[tensor_id=267 [16x197x12x197] FP32  nslice=1 hslice=12]</td>\n",
       "      <td>246017536</td>\n",
       "      <td>326713344</td>\n",
       "      <td>564.736</td>\n",
       "      <td>1.624%</td>\n",
       "      <td>240.2515</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>75.301%</td>\n",
       "      <td>337</td>\n",
       "      <td>local</td>\n",
       "      <td>{'FP32'}</td>\n",
       "      <td>Softmax</td>\n",
       "      <td>ins=[tensor_id=336 [16x197x12x197] FP32  nslice=1 hslice=12],outs=[tensor_id=337 [16x197x12x197] FP32  nslice=1 hslice=12]</td>\n",
       "      <td>246017536</td>\n",
       "      <td>326713344</td>\n",
       "      <td>564.736</td>\n",
       "      <td>1.624%</td>\n",
       "      <td>240.2515</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>75.301%</td>\n",
       "      <td>407</td>\n",
       "      <td>local</td>\n",
       "      <td>{'FP32'}</td>\n",
       "      <td>Softmax</td>\n",
       "      <td>ins=[tensor_id=406 [16x197x12x197] FP32  nslice=1 hslice=12],outs=[tensor_id=407 [16x197x12x197] FP32  nslice=1 hslice=12]</td>\n",
       "      <td>246017536</td>\n",
       "      <td>326713344</td>\n",
       "      <td>564.736</td>\n",
       "      <td>1.624%</td>\n",
       "      <td>240.2515</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>75.301%</td>\n",
       "      <td>617</td>\n",
       "      <td>local</td>\n",
       "      <td>{'FP32'}</td>\n",
       "      <td>Softmax</td>\n",
       "      <td>ins=[tensor_id=616 [16x197x12x197] FP32  nslice=1 hslice=12],outs=[tensor_id=617 [16x197x12x197] FP32  nslice=1 hslice=12]</td>\n",
       "      <td>246017536</td>\n",
       "      <td>326713344</td>\n",
       "      <td>564.736</td>\n",
       "      <td>1.624%</td>\n",
       "      <td>240.2515</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>75.301%</td>\n",
       "      <td>757</td>\n",
       "      <td>local</td>\n",
       "      <td>{'FP32'}</td>\n",
       "      <td>Softmax</td>\n",
       "      <td>ins=[tensor_id=756 [16x197x12x197] FP32  nslice=1 hslice=12],outs=[tensor_id=757 [16x197x12x197] FP32  nslice=1 hslice=12]</td>\n",
       "      <td>246017536</td>\n",
       "      <td>326713344</td>\n",
       "      <td>564.736</td>\n",
       "      <td>1.624%</td>\n",
       "      <td>240.2515</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>75.301%</td>\n",
       "      <td>127</td>\n",
       "      <td>local</td>\n",
       "      <td>{'FP32'}</td>\n",
       "      <td>Softmax</td>\n",
       "      <td>ins=[tensor_id=126 [16x197x12x197] FP32  nslice=1 hslice=12],outs=[tensor_id=127 [16x197x12x197] FP32  nslice=1 hslice=12]</td>\n",
       "      <td>246017536</td>\n",
       "      <td>326713344</td>\n",
       "      <td>564.736</td>\n",
       "      <td>1.624%</td>\n",
       "      <td>240.2515</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>75.301%</td>\n",
       "      <td>62</td>\n",
       "      <td>local</td>\n",
       "      <td>{'FP32'}</td>\n",
       "      <td>Softmax</td>\n",
       "      <td>ins=[tensor_id=60 [16x197x12x197] FP32  nslice=1 hslice=12],outs=[tensor_id=62 [16x197x12x197] FP32  nslice=1 hslice=12]</td>\n",
       "      <td>246017536</td>\n",
       "      <td>326713344</td>\n",
       "      <td>564.736</td>\n",
       "      <td>1.624%</td>\n",
       "      <td>240.2515</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>75.301%</td>\n",
       "      <td>477</td>\n",
       "      <td>local</td>\n",
       "      <td>{'FP32'}</td>\n",
       "      <td>Softmax</td>\n",
       "      <td>ins=[tensor_id=476 [16x197x12x197] FP32  nslice=1 hslice=12],outs=[tensor_id=477 [16x197x12x197] FP32  nslice=1 hslice=12]</td>\n",
       "      <td>246017536</td>\n",
       "      <td>326713344</td>\n",
       "      <td>564.736</td>\n",
       "      <td>1.624%</td>\n",
       "      <td>240.2515</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>75.301%</td>\n",
       "      <td>827</td>\n",
       "      <td>local</td>\n",
       "      <td>{'FP32'}</td>\n",
       "      <td>Softmax</td>\n",
       "      <td>ins=[tensor_id=826 [16x197x12x197] FP32  nslice=1 hslice=12],outs=[tensor_id=827 [16x197x12x197] FP32  nslice=1 hslice=12]</td>\n",
       "      <td>246017536</td>\n",
       "      <td>326713344</td>\n",
       "      <td>564.736</td>\n",
       "      <td>1.624%</td>\n",
       "      <td>240.2515</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>75.301%</td>\n",
       "      <td>197</td>\n",
       "      <td>local</td>\n",
       "      <td>{'FP32'}</td>\n",
       "      <td>Softmax</td>\n",
       "      <td>ins=[tensor_id=196 [16x197x12x197] FP32  nslice=1 hslice=12],outs=[tensor_id=197 [16x197x12x197] FP32  nslice=1 hslice=12]</td>\n",
       "      <td>246017536</td>\n",
       "      <td>326713344</td>\n",
       "      <td>564.736</td>\n",
       "      <td>1.624%</td>\n",
       "      <td>240.2515</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>75.301%</td>\n",
       "      <td>547</td>\n",
       "      <td>local</td>\n",
       "      <td>{'FP32'}</td>\n",
       "      <td>Softmax</td>\n",
       "      <td>ins=[tensor_id=546 [16x197x12x197] FP32  nslice=1 hslice=12],outs=[tensor_id=547 [16x197x12x197] FP32  nslice=1 hslice=12]</td>\n",
       "      <td>246017536</td>\n",
       "      <td>326713344</td>\n",
       "      <td>564.512</td>\n",
       "      <td>1.623%</td>\n",
       "      <td>240.2515</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>75.301%</td>\n",
       "      <td>687</td>\n",
       "      <td>local</td>\n",
       "      <td>{'FP32'}</td>\n",
       "      <td>Softmax</td>\n",
       "      <td>ins=[tensor_id=686 [16x197x12x197] FP32  nslice=1 hslice=12],outs=[tensor_id=687 [16x197x12x197] FP32  nslice=1 hslice=12]</td>\n",
       "      <td>246017536</td>\n",
       "      <td>326713344</td>\n",
       "      <td>564.512</td>\n",
       "      <td>1.623%</td>\n",
       "      <td>240.2515</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   uArchRate  LayerID   Type  DataType Function                                                                                                                  Other info     AlgOps   uArchOps  tiuTime(us) tiuTimeRatio  tiuPTheoTime(us)  totalTime(us)  PeakTops  ActualTops Concurrency\n",
       "0    75.301%      267  local  {'FP32'}  Softmax  ins=[tensor_id=266 [16x197x12x197] FP32  nslice=1 hslice=12],outs=[tensor_id=267 [16x197x12x197] FP32  nslice=1 hslice=12]  246017536  326713344      564.736       1.624%          240.2515            NaN       1.0         NaN         NaN\n",
       "1    75.301%      337  local  {'FP32'}  Softmax  ins=[tensor_id=336 [16x197x12x197] FP32  nslice=1 hslice=12],outs=[tensor_id=337 [16x197x12x197] FP32  nslice=1 hslice=12]  246017536  326713344      564.736       1.624%          240.2515            NaN       1.0         NaN         NaN\n",
       "2    75.301%      407  local  {'FP32'}  Softmax  ins=[tensor_id=406 [16x197x12x197] FP32  nslice=1 hslice=12],outs=[tensor_id=407 [16x197x12x197] FP32  nslice=1 hslice=12]  246017536  326713344      564.736       1.624%          240.2515            NaN       1.0         NaN         NaN\n",
       "3    75.301%      617  local  {'FP32'}  Softmax  ins=[tensor_id=616 [16x197x12x197] FP32  nslice=1 hslice=12],outs=[tensor_id=617 [16x197x12x197] FP32  nslice=1 hslice=12]  246017536  326713344      564.736       1.624%          240.2515            NaN       1.0         NaN         NaN\n",
       "4    75.301%      757  local  {'FP32'}  Softmax  ins=[tensor_id=756 [16x197x12x197] FP32  nslice=1 hslice=12],outs=[tensor_id=757 [16x197x12x197] FP32  nslice=1 hslice=12]  246017536  326713344      564.736       1.624%          240.2515            NaN       1.0         NaN         NaN\n",
       "5    75.301%      127  local  {'FP32'}  Softmax  ins=[tensor_id=126 [16x197x12x197] FP32  nslice=1 hslice=12],outs=[tensor_id=127 [16x197x12x197] FP32  nslice=1 hslice=12]  246017536  326713344      564.736       1.624%          240.2515            NaN       1.0         NaN         NaN\n",
       "6    75.301%       62  local  {'FP32'}  Softmax    ins=[tensor_id=60 [16x197x12x197] FP32  nslice=1 hslice=12],outs=[tensor_id=62 [16x197x12x197] FP32  nslice=1 hslice=12]  246017536  326713344      564.736       1.624%          240.2515            NaN       1.0         NaN         NaN\n",
       "7    75.301%      477  local  {'FP32'}  Softmax  ins=[tensor_id=476 [16x197x12x197] FP32  nslice=1 hslice=12],outs=[tensor_id=477 [16x197x12x197] FP32  nslice=1 hslice=12]  246017536  326713344      564.736       1.624%          240.2515            NaN       1.0         NaN         NaN\n",
       "8    75.301%      827  local  {'FP32'}  Softmax  ins=[tensor_id=826 [16x197x12x197] FP32  nslice=1 hslice=12],outs=[tensor_id=827 [16x197x12x197] FP32  nslice=1 hslice=12]  246017536  326713344      564.736       1.624%          240.2515            NaN       1.0         NaN         NaN\n",
       "9    75.301%      197  local  {'FP32'}  Softmax  ins=[tensor_id=196 [16x197x12x197] FP32  nslice=1 hslice=12],outs=[tensor_id=197 [16x197x12x197] FP32  nslice=1 hslice=12]  246017536  326713344      564.736       1.624%          240.2515            NaN       1.0         NaN         NaN\n",
       "10   75.301%      547  local  {'FP32'}  Softmax  ins=[tensor_id=546 [16x197x12x197] FP32  nslice=1 hslice=12],outs=[tensor_id=547 [16x197x12x197] FP32  nslice=1 hslice=12]  246017536  326713344      564.512       1.623%          240.2515            NaN       1.0         NaN         NaN\n",
       "11   75.301%      687  local  {'FP32'}  Softmax  ins=[tensor_id=686 [16x197x12x197] FP32  nslice=1 hslice=12],outs=[tensor_id=687 [16x197x12x197] FP32  nslice=1 hslice=12]  246017536  326713344      564.512       1.623%          240.2515            NaN       1.0         NaN         NaN"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "softmax = get_layers(layer_df, 'Softmax')\n",
    "softmax.to_csv(profile_dir + 'softmax.csv')\n",
    "softmax\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3. ddrRate优化"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## ddrRate低的问题需要找到对应的指令，检查是否有不满足下述限制的情况：\n",
    "### (1) 4DDR-interleave：DDR0，DDR1，DDRA，DDRB, 每块DDR连续4KiB数据\n",
    "检查搬运的数据是否用到了每个DDR，如果只用到了2个，则性能最多只有1/2，即30GiB/s。\n",
    "### (2) 地址对齐：32B，64B\n",
    "指令的src_addr和dst_addr\n",
    "### (3) 和TIU指令bank冲突\n",
    "### (4) 数据量：大于64KB\n",
    "### (5) 最小连续的长度：大于256B\n",
    "### (6) stride\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ddrRate\n",
       "26.958%    997.675\n",
       "92.537%    222.257\n",
       "94.151%    105.333\n",
       "92.117%     84.394\n",
       "47.372%     79.971\n",
       "47.424%     79.884\n",
       "47.534%     79.698\n",
       "47.568%     79.641\n",
       "47.579%     79.623\n",
       "47.631%     79.537\n",
       "47.816%     79.229\n",
       "47.849%     79.174\n",
       "47.866%     79.145\n",
       "47.950%     79.007\n",
       "48.050%     78.843\n",
       "53.396%     70.949\n",
       "92.005%     62.836\n",
       "84.058%     60.956\n",
       "85.183%     60.151\n",
       "85.742%     59.759\n",
       "89.374%     57.330\n",
       "89.900%     56.995\n",
       "89.901%     56.994\n",
       "90.234%     56.784\n",
       "90.396%     56.682\n",
       "90.506%     56.613\n",
       "90.537%     56.594\n",
       "90.842%     56.404\n",
       "91.024%     56.291\n",
       "92.911%     55.148\n",
       "94.450%     54.249\n",
       "94.805%     54.046\n",
       "95.007%     53.931\n",
       "95.042%     53.911\n",
       "95.085%     53.887\n",
       "95.427%     53.694\n",
       "95.556%     53.621\n",
       "95.653%     53.567\n",
       "95.705%     53.538\n",
       "95.755%     53.510\n",
       "95.801%     53.484\n",
       "78.147%     49.740\n",
       "79.076%     49.156\n",
       "79.156%     49.106\n",
       "79.456%     48.921\n",
       "79.503%     48.892\n",
       "79.769%     48.729\n",
       "79.921%     48.636\n",
       "79.954%     48.616\n",
       "80.467%     48.306\n",
       "80.544%     48.260\n",
       "80.782%     48.118\n",
       "83.095%     46.541\n",
       "83.988%     46.281\n",
       "86.465%     44.955\n",
       "88.991%     43.679\n",
       "89.010%     43.670\n",
       "89.976%     43.201\n",
       "90.061%     43.160\n",
       "90.535%     42.934\n",
       "90.706%     42.853\n",
       "90.987%     42.721\n",
       "91.027%     42.702\n",
       "91.215%     42.614\n",
       "91.301%     42.574\n",
       "91.471%     42.495\n",
       "91.494%     42.484\n",
       "91.503%     42.480\n",
       "91.903%     42.295\n",
       "92.187%     42.165\n",
       "92.412%     42.062\n",
       "92.896%     41.843\n",
       "93.241%     41.688\n",
       "90.951%     41.653\n",
       "92.605%     40.909\n",
       "92.617%     40.904\n",
       "93.326%     40.593\n",
       "93.584%     40.481\n",
       "93.742%     40.413\n",
       "93.993%     40.305\n",
       "94.276%     40.184\n",
       "94.356%     40.150\n",
       "94.462%     40.105\n",
       "94.606%     40.044\n",
       "94.990%     39.882\n",
       "53.904%     23.991\n",
       "90.649%     20.896\n",
       "84.827%     11.165\n",
       "87.784%     10.789\n",
       "88.233%     10.734\n",
       "88.605%     10.689\n",
       "89.704%     10.558\n",
       "89.968%     10.527\n",
       "90.312%     10.487\n",
       "90.363%     10.481\n",
       "90.718%     10.440\n",
       "90.805%     10.430\n",
       "90.857%     10.424\n",
       "90.936%     10.415\n",
       "90.997%     10.408\n",
       "91.392%     10.363\n",
       "91.463%     10.355\n",
       "91.480%     10.353\n",
       "91.622%     10.337\n",
       "91.693%     10.329\n",
       "91.817%     10.315\n",
       "91.933%     10.302\n",
       "92.085%     10.285\n",
       "92.355%     10.255\n",
       "92.418%     10.248\n",
       "92.744%     10.212\n",
       "92.907%     10.194\n",
       "92.989%     10.185\n",
       "93.209%     10.161\n",
       "93.448%     10.135\n",
       "93.467%     10.133\n",
       "93.707%     10.107\n",
       "94.042%     10.071\n",
       "94.145%     10.060\n",
       "94.483%     10.024\n",
       "94.540%     10.018\n",
       "94.653%     10.006\n",
       "94.786%      9.992\n",
       "95.014%      9.968\n",
       "95.081%      9.961\n",
       "95.109%      9.958\n",
       "95.119%      9.957\n",
       "95.138%      9.955\n",
       "95.889%      9.877\n",
       "96.103%      9.855\n",
       "96.289%      9.836\n",
       "35.538%      6.836\n",
       "13.895%      5.680\n",
       "13.404%      5.520\n",
       "8.208%       4.808\n",
       "13.934%      3.540\n",
       "13.779%      3.222\n",
       "13.856%      3.204\n",
       "38.165%      3.102\n",
       "62.055%      2.597\n",
       "13.817%      1.785\n",
       "14.014%      1.760\n",
       "13.478%      1.098\n",
       "38.239%      1.032\n",
       "12.981%      0.760\n",
       "13.084%      0.754\n",
       "13.189%      0.748\n",
       "26.379%      0.748\n",
       "13.260%      0.744\n",
       "13.296%      0.742\n",
       "13.515%      0.730\n",
       "13.552%      0.728\n",
       "13.627%      0.724\n",
       "13.974%      0.706\n",
       "7.149%       0.690\n",
       "62.400%      0.655\n",
       "8.793%       0.561\n",
       "8.824%       0.559\n",
       "8.840%       0.558\n",
       "2.271%       0.543\n",
       "0.774%       0.531\n",
       "9.307%       0.530\n",
       "37.370%      0.528\n",
       "37.945%      0.520\n",
       "0.795%       0.517\n",
       "38.388%      0.514\n",
       "9.787%       0.504\n",
       "10.026%      0.492\n",
       "10.255%      0.481\n",
       "11.340%      0.435\n",
       "12.271%      0.402\n",
       "12.301%      0.401\n",
       "12.779%      0.386\n",
       "12.812%      0.385\n",
       "13.119%      0.376\n",
       "13.332%      0.370\n",
       "13.368%      0.369\n",
       "13.441%      0.367\n",
       "2.246%       0.366\n",
       "13.589%      0.363\n",
       "2.284%       0.360\n",
       "13.740%      0.359\n",
       "2.322%       0.354\n",
       "14.054%      0.351\n",
       "2.309%       0.178\n",
       "--           0.000\n",
       "Name: gdmaTime(us), dtype: float64"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "layer_df.groupby('ddrRate')['gdmaTime(us)'].sum().sort_values(ascending=False)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 4 Concurrency优化"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 每个GlobalOp的Concurrency\n",
    "部分Op出现Concurrency为负数的情况，这是由于指令之间存在间隙导致totalTime > tiuTime + gdmaTime\n",
    "\n",
    "**注：指令间隙通常是因为之前的指令执行时间过短，导致后续指令尚未填充入指令buffer导致的，另外，如果是动态网络，由于指令是由CPU实时产生发送的，这种情况会更频繁。**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>LayerID</th>\n",
       "      <th>Type</th>\n",
       "      <th>TPU/CPU</th>\n",
       "      <th>DataType</th>\n",
       "      <th>Function</th>\n",
       "      <th>in</th>\n",
       "      <th>ic</th>\n",
       "      <th>ih</th>\n",
       "      <th>iw</th>\n",
       "      <th>on</th>\n",
       "      <th>oc</th>\n",
       "      <th>oh</th>\n",
       "      <th>ow</th>\n",
       "      <th>kh</th>\n",
       "      <th>kw</th>\n",
       "      <th>KStrideH</th>\n",
       "      <th>KStrideW</th>\n",
       "      <th>Padding</th>\n",
       "      <th>Other info</th>\n",
       "      <th>inputBytes</th>\n",
       "      <th>outputBytes</th>\n",
       "      <th>weightBytes</th>\n",
       "      <th>s2lBytes</th>\n",
       "      <th>l2sBytes</th>\n",
       "      <th>s2sBytes</th>\n",
       "      <th>gdmaCycles</th>\n",
       "      <th>gdmaTime(us)</th>\n",
       "      <th>gdmaTimeRatio</th>\n",
       "      <th>gdmaPTheoTime(us)</th>\n",
       "      <th>ddrRate</th>\n",
       "      <th>LoadAvgBandwidth(GiB/s)</th>\n",
       "      <th>StoreAvgBandwidth(GiB/s)</th>\n",
       "      <th>AlgOps</th>\n",
       "      <th>uArchOps</th>\n",
       "      <th>uArchCModelCycles</th>\n",
       "      <th>uArchCModelCycleRatio</th>\n",
       "      <th>tiuCycles</th>\n",
       "      <th>tiuTime(us)</th>\n",
       "      <th>tiuTimeRatio</th>\n",
       "      <th>tiuPTheoTime(us)</th>\n",
       "      <th>uArchRate</th>\n",
       "      <th>totalTime(us)</th>\n",
       "      <th>PeakTops</th>\n",
       "      <th>ActualTops</th>\n",
       "      <th>Parallelism</th>\n",
       "      <th>Concurrency</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>97</th>\n",
       "      <td>19</td>\n",
       "      <td>global</td>\n",
       "      <td>TPU</td>\n",
       "      <td>{'INT8'}</td>\n",
       "      <td>MatMul</td>\n",
       "      <td>16</td>\n",
       "      <td>196</td>\n",
       "      <td>768</td>\n",
       "      <td>1</td>\n",
       "      <td>16</td>\n",
       "      <td>196</td>\n",
       "      <td>768</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>[0, 0, 0, 0]</td>\n",
       "      <td>ins=[tensor_id=16 [16x196x768] INT8  ,tensor_id=17 [1x768x768] INT8 CONST ,tensor_id=18 [1x1x768] INT32 CONST ],outs=[tensor_id=19 [16x196x768] INT8  ]</td>\n",
       "      <td>2408448</td>\n",
       "      <td>2408448</td>\n",
       "      <td>592896</td>\n",
       "      <td>3001344</td>\n",
       "      <td>2408448</td>\n",
       "      <td>0</td>\n",
       "      <td>105333</td>\n",
       "      <td>105.333</td>\n",
       "      <td>1.687%</td>\n",
       "      <td>99.171665</td>\n",
       "      <td>94.151%</td>\n",
       "      <td>53.830825</td>\n",
       "      <td>41.999026</td>\n",
       "      <td>3709009920</td>\n",
       "      <td>3709009920</td>\n",
       "      <td>122304</td>\n",
       "      <td>0.388%</td>\n",
       "      <td>133630</td>\n",
       "      <td>133.630</td>\n",
       "      <td>0.384%</td>\n",
       "      <td>113.190000</td>\n",
       "      <td>100.000%</td>\n",
       "      <td>172.730</td>\n",
       "      <td>32.0</td>\n",
       "      <td>21.472876</td>\n",
       "      <td>138.345%</td>\n",
       "      <td>62.880%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>159</th>\n",
       "      <td>13</td>\n",
       "      <td>global</td>\n",
       "      <td>TPU</td>\n",
       "      <td>{'FP32'}</td>\n",
       "      <td>Cast</td>\n",
       "      <td>16</td>\n",
       "      <td>3</td>\n",
       "      <td>14</td>\n",
       "      <td>16</td>\n",
       "      <td>16</td>\n",
       "      <td>3</td>\n",
       "      <td>14</td>\n",
       "      <td>16</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>[0, 0, 0, 0]</td>\n",
       "      <td>ins=[tensor_id=12 [16x3x14x16x14x16] FP32  ],outs=[tensor_id=13 [16x3x14x16x14x16] INT8  ]</td>\n",
       "      <td>9633792</td>\n",
       "      <td>2408448</td>\n",
       "      <td>0</td>\n",
       "      <td>9633792</td>\n",
       "      <td>2408448</td>\n",
       "      <td>0</td>\n",
       "      <td>222257</td>\n",
       "      <td>222.257</td>\n",
       "      <td>3.559%</td>\n",
       "      <td>205.670778</td>\n",
       "      <td>92.537%</td>\n",
       "      <td>53.264990</td>\n",
       "      <td>41.682158</td>\n",
       "      <td>9633792</td>\n",
       "      <td>9699328</td>\n",
       "      <td>14208</td>\n",
       "      <td>0.045%</td>\n",
       "      <td>14269</td>\n",
       "      <td>14.269</td>\n",
       "      <td>0.041%</td>\n",
       "      <td>9.408000</td>\n",
       "      <td>99.324%</td>\n",
       "      <td>222.285</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.043340</td>\n",
       "      <td>106.407%</td>\n",
       "      <td>99.804%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>160</th>\n",
       "      <td>15</td>\n",
       "      <td>global</td>\n",
       "      <td>TPU</td>\n",
       "      <td>{'INT8'}</td>\n",
       "      <td>Permute</td>\n",
       "      <td>16</td>\n",
       "      <td>3</td>\n",
       "      <td>14</td>\n",
       "      <td>16</td>\n",
       "      <td>16</td>\n",
       "      <td>14</td>\n",
       "      <td>14</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>[0, 0, 0, 0]</td>\n",
       "      <td>ins=[tensor_id=13 [16x3x14x16x14x16] INT8  ,tensor_id=14 [2408448] INT8  ],outs=[tensor_id=15 [16x14x14x3x16x16] INT8  ]</td>\n",
       "      <td>4816896</td>\n",
       "      <td>2408448</td>\n",
       "      <td>0</td>\n",
       "      <td>7225344</td>\n",
       "      <td>7225344</td>\n",
       "      <td>0</td>\n",
       "      <td>997675</td>\n",
       "      <td>997.675</td>\n",
       "      <td>15.976%</td>\n",
       "      <td>268.954095</td>\n",
       "      <td>26.958%</td>\n",
       "      <td>8.056556</td>\n",
       "      <td>41.425557</td>\n",
       "      <td>7225344</td>\n",
       "      <td>7585792</td>\n",
       "      <td>55360</td>\n",
       "      <td>0.176%</td>\n",
       "      <td>12483</td>\n",
       "      <td>12.483</td>\n",
       "      <td>0.036%</td>\n",
       "      <td>1.764000</td>\n",
       "      <td>95.248%</td>\n",
       "      <td>1010.184</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.007153</td>\n",
       "      <td>99.997%</td>\n",
       "      <td>-0.208%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>250</th>\n",
       "      <td>878</td>\n",
       "      <td>global</td>\n",
       "      <td>TPU</td>\n",
       "      <td>{'INT8'}</td>\n",
       "      <td>MatMul</td>\n",
       "      <td>16</td>\n",
       "      <td>768</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>16</td>\n",
       "      <td>1000</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>[0, 0, 0, 0]</td>\n",
       "      <td>ins=[tensor_id=875 [16x768] INT8  ,tensor_id=876 [768x1000] INT8 CONST ,tensor_id=877 [1x1000] INT32 CONST ],outs=[tensor_id=878 [16x1000] INT8  ]</td>\n",
       "      <td>12288</td>\n",
       "      <td>16000</td>\n",
       "      <td>772000</td>\n",
       "      <td>784288</td>\n",
       "      <td>16000</td>\n",
       "      <td>0</td>\n",
       "      <td>23991</td>\n",
       "      <td>23.991</td>\n",
       "      <td>0.384%</td>\n",
       "      <td>12.932199</td>\n",
       "      <td>53.904%</td>\n",
       "      <td>31.274893</td>\n",
       "      <td>23.429499</td>\n",
       "      <td>24640000</td>\n",
       "      <td>98562048</td>\n",
       "      <td>3252</td>\n",
       "      <td>0.010%</td>\n",
       "      <td>3483</td>\n",
       "      <td>3.483</td>\n",
       "      <td>0.010%</td>\n",
       "      <td>0.751953</td>\n",
       "      <td>24.999%</td>\n",
       "      <td>27.490</td>\n",
       "      <td>32.0</td>\n",
       "      <td>0.896326</td>\n",
       "      <td>99.942%</td>\n",
       "      <td>-0.459%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>326</th>\n",
       "      <td>874</td>\n",
       "      <td>global</td>\n",
       "      <td>TPU</td>\n",
       "      <td>{'INT8'}</td>\n",
       "      <td>Slice</td>\n",
       "      <td>16</td>\n",
       "      <td>197</td>\n",
       "      <td>768</td>\n",
       "      <td>1</td>\n",
       "      <td>16</td>\n",
       "      <td>1</td>\n",
       "      <td>768</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>[0, 0, 0, 0]</td>\n",
       "      <td>ins=[tensor_id=871 [16x197x768] INT8  nslice=1 hslice=768],outs=[tensor_id=874 [16x1x768] INT8  ]</td>\n",
       "      <td>2420736</td>\n",
       "      <td>12288</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>12288</td>\n",
       "      <td>655</td>\n",
       "      <td>0.655</td>\n",
       "      <td>0.010%</td>\n",
       "      <td>0.408718</td>\n",
       "      <td>62.400%</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000%</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000%</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>--</td>\n",
       "      <td>0.655</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>100.000%</td>\n",
       "      <td>100%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>568</th>\n",
       "      <td>879</td>\n",
       "      <td>global</td>\n",
       "      <td>TPU</td>\n",
       "      <td>{'INT8'}</td>\n",
       "      <td>Cast</td>\n",
       "      <td>16</td>\n",
       "      <td>1000</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>16</td>\n",
       "      <td>1000</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>[0, 0, 0, 0]</td>\n",
       "      <td>ins=[tensor_id=878 [16x1000] INT8  ],outs=[tensor_id=879 [16x1000] FP32  ]</td>\n",
       "      <td>16000</td>\n",
       "      <td>64000</td>\n",
       "      <td>0</td>\n",
       "      <td>16000</td>\n",
       "      <td>64000</td>\n",
       "      <td>0</td>\n",
       "      <td>2597</td>\n",
       "      <td>2.597</td>\n",
       "      <td>0.042%</td>\n",
       "      <td>1.611568</td>\n",
       "      <td>62.055%</td>\n",
       "      <td>21.076607</td>\n",
       "      <td>31.536849</td>\n",
       "      <td>48000</td>\n",
       "      <td>49152</td>\n",
       "      <td>16</td>\n",
       "      <td>0.000%</td>\n",
       "      <td>29</td>\n",
       "      <td>0.029</td>\n",
       "      <td>0.000%</td>\n",
       "      <td>0.011719</td>\n",
       "      <td>97.656%</td>\n",
       "      <td>2.632</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.018237</td>\n",
       "      <td>99.772%</td>\n",
       "      <td>-20.690%</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     LayerID    Type TPU/CPU  DataType Function  in    ic   ih  iw  on    oc   oh  ow  kh  kw  KStrideH  KStrideW       Padding                                                                                                                                               Other info  inputBytes  outputBytes  weightBytes  s2lBytes  l2sBytes  s2sBytes  gdmaCycles  gdmaTime(us) gdmaTimeRatio  gdmaPTheoTime(us)  ddrRate  LoadAvgBandwidth(GiB/s)  StoreAvgBandwidth(GiB/s)      AlgOps    uArchOps  uArchCModelCycles uArchCModelCycleRatio  tiuCycles  tiuTime(us) tiuTimeRatio  tiuPTheoTime(us) uArchRate  totalTime(us)  PeakTops  ActualTops Parallelism Concurrency\n",
       "97        19  global     TPU  {'INT8'}   MatMul  16   196  768   1  16   196  768   1   0   0         0         0  [0, 0, 0, 0]  ins=[tensor_id=16 [16x196x768] INT8  ,tensor_id=17 [1x768x768] INT8 CONST ,tensor_id=18 [1x1x768] INT32 CONST ],outs=[tensor_id=19 [16x196x768] INT8  ]     2408448      2408448       592896   3001344   2408448         0      105333       105.333        1.687%          99.171665  94.151%                53.830825                 41.999026  3709009920  3709009920             122304                0.388%     133630      133.630       0.384%        113.190000  100.000%        172.730      32.0   21.472876    138.345%     62.880%\n",
       "159       13  global     TPU  {'FP32'}     Cast  16     3   14  16  16     3   14  16   0   0         0         0  [0, 0, 0, 0]                                                               ins=[tensor_id=12 [16x3x14x16x14x16] FP32  ],outs=[tensor_id=13 [16x3x14x16x14x16] INT8  ]     9633792      2408448            0   9633792   2408448         0      222257       222.257        3.559%         205.670778  92.537%                53.264990                 41.682158     9633792     9699328              14208                0.045%      14269       14.269       0.041%          9.408000   99.324%        222.285       1.0    0.043340    106.407%     99.804%\n",
       "160       15  global     TPU  {'INT8'}  Permute  16     3   14  16  16    14   14   3   0   0         0         0  [0, 0, 0, 0]                                 ins=[tensor_id=13 [16x3x14x16x14x16] INT8  ,tensor_id=14 [2408448] INT8  ],outs=[tensor_id=15 [16x14x14x3x16x16] INT8  ]     4816896      2408448            0   7225344   7225344         0      997675       997.675       15.976%         268.954095  26.958%                 8.056556                 41.425557     7225344     7585792              55360                0.176%      12483       12.483       0.036%          1.764000   95.248%       1010.184       4.0    0.007153     99.997%     -0.208%\n",
       "250      878  global     TPU  {'INT8'}   MatMul  16   768    1   1  16  1000    1   1   0   0         0         0  [0, 0, 0, 0]       ins=[tensor_id=875 [16x768] INT8  ,tensor_id=876 [768x1000] INT8 CONST ,tensor_id=877 [1x1000] INT32 CONST ],outs=[tensor_id=878 [16x1000] INT8  ]       12288        16000       772000    784288     16000         0       23991        23.991        0.384%          12.932199  53.904%                31.274893                 23.429499    24640000    98562048               3252                0.010%       3483        3.483       0.010%          0.751953   24.999%         27.490      32.0    0.896326     99.942%     -0.459%\n",
       "326      874  global     TPU  {'INT8'}    Slice  16   197  768   1  16     1  768   1   0   0         0         0  [0, 0, 0, 0]                                                        ins=[tensor_id=871 [16x197x768] INT8  nslice=1 hslice=768],outs=[tensor_id=874 [16x1x768] INT8  ]     2420736        12288            0         0         0     12288         655         0.655        0.010%           0.408718  62.400%                 0.000000                  0.000000           0           0                  0                0.000%          0        0.000       0.000%          0.000000        --          0.655       4.0    0.000000    100.000%        100%\n",
       "568      879  global     TPU  {'INT8'}     Cast  16  1000    1   1  16  1000    1   1   0   0         0         0  [0, 0, 0, 0]                                                                               ins=[tensor_id=878 [16x1000] INT8  ],outs=[tensor_id=879 [16x1000] FP32  ]       16000        64000            0     16000     64000         0        2597         2.597        0.042%           1.611568  62.055%                21.076607                 31.536849       48000       49152                 16                0.000%         29        0.029       0.000%          0.011719   97.656%          2.632       4.0    0.018237     99.772%    -20.690%"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "layer_df.loc[layer_df['Type'] == 'global']\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 5. macUtil分析"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 上限1\n",
    "用MacUtilUpperLimit表示：tiuModelTheoTime / sum(tiuPTheoTime)，其中tiuModelTheoTime = totalAlgOps / PeakTops，tiuLayerTheoTime = layerAlgOps / LayerPeakTops\n",
    "1. 反映了不同LayerType的PeakTops的差异对于计算macUtil的影响，这个上限很难提升\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "summary_df.set_index('Function', inplace=True)\n",
    "summary_df['uArchRate'] = summary_df['uArchRate'].apply(lambda x: float(x.strip('%')) if x != '--' else x)\n",
    "summary_df['Concurrency'] = summary_df['Concurrency'].apply(lambda x: float(x.strip('%')) if x != '--' else x)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tiuModelTheoTime = 17216.50 us\n",
      "tiuModelPTheoTime = 22877.62 us \n",
      "MacUtilUpperLimit = 75.25%\n"
     ]
    }
   ],
   "source": [
    "# ModelAlgOps来自final.mlir中，这是模型原始的计算量，profile中的AlgOps是原本算子拆成多个指令后的，指令的有效计算量，与实现相关\n",
    "# ModelAlgOps = 564150211456\n",
    "tiuModelTheoTime = ModelAlgOps / ModelPeakTops * 1e6 # us\n",
    "tiuModelPTheoTime = summary_df.at['Overall', 'tiuPTheoTime(us)'] # us\n",
    "MacUtilUpperLimit = tiuModelTheoTime / tiuModelPTheoTime\n",
    "\n",
    "print(f\"tiuModelTheoTime = {tiuModelTheoTime:.2f} us\\ntiuModelPTheoTime = {tiuModelPTheoTime:.2f} us \\nMacUtilUpperLimit = {MacUtilUpperLimit*100:.2f}%\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 上限2\n",
    "用MaxMacUtil表示：tiuModelTheoTime / MAX(gdmaTime, tiuTime)\n",
    "1. 反映了并行度打满的情况下macUtil的上限\n",
    "2. 如果这时候macUtil依然达不到期望，那就需要优化MAX(gdmaTime, tiuTime)，通常是tiuTime，因为如果gdmaTime更长，一般这会是带宽瓶颈，从ddrUtil角度优化。\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 端到端runtime耗时，e2eTime来自bmrt_test的calculate time\n",
    "# e2eTime = 38300. # us\n",
    "macUtil0 = tiuModelTheoTime / e2eTime\n",
    "\n",
    "# 模型纯粹的运行耗时，不考虑CPU耗时及编译空间的输入输出的搬运\n",
    "totalModelTime = summary_df.at['Overall', 'totalTime(us)']\n",
    "macUtil1 = tiuModelTheoTime / totalModelTime\n",
    "\n",
    "# 并行度打满\n",
    "tiuModelTime = summary_df.at['Overall', 'tiuTime(us)']\n",
    "macUtil2 = tiuModelTheoTime / tiuModelTime\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### macUtil影响因素分析\n",
    "1. e2eTime\n",
    "2. Concurrency\n",
    "3. 各个算子的uArchRate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_mac_util1_with_tiu_theo_time(tiuModelTheoTime, tiuTotalTime, replace_layer_type):\n",
    "    alg_ops = summary_df.at[replace_layer_type, 'AlgOps']\n",
    "    peak_tops = summary_df.at[\"Overall\", \"PeakTops\"]\n",
    "    tiu_time = summary_df.at[replace_layer_type, 'tiuTime(us)']\n",
    "    tiu_theo_time = alg_ops / (peak_tops * 1024 * tpu_freq) * 1e6\n",
    "    reduced_time = tiu_time - tiu_theo_time\n",
    "    tiuTotalTime = tiuTotalTime - reduced_time\n",
    "    macUtil = tiuModelTheoTime / tiuTotalTime\n",
    "    return [replace_layer_type + f' tiuTime: {tiu_time:.2f}us -> {tiu_theo_time:.2f}us', reduced_time, tiuTotalTime , 100.00, macUtil*100, f'{replace_layer_type}的耗时用ModelPeakTops得到的理论耗时替换']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_mac_util2_with_tiu_ptheo_time(tiuModelTheoTime, tiuTotalTime, replace_layer_type):\n",
    "    tiuTime = summary_df.at[replace_layer_type, 'tiuTime(us)']\n",
    "    tiuPTheoTime = summary_df.at[replace_layer_type, 'tiuPTheoTime(us)']\n",
    "    reduced_time =  tiuTime - tiuPTheoTime\n",
    "    tiuTotalTime = tiuTotalTime - reduced_time\n",
    "    macUtil = tiuModelTheoTime / tiuTotalTime\n",
    "    return [replace_layer_type + f' tiuTime: {tiuTime:.2f}us -> {tiuPTheoTime:.2f}us', reduced_time, tiuTotalTime , 100.00, macUtil*100, f'{replace_layer_type}的耗时用LayerPeakTops得到的理论耗时替换']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_mac_util3_with_full_uarch_rate(tiuModelTheoTime, tiuTotalTime, replace_layer_type):\n",
    "    uArchRate = summary_df.at[replace_layer_type, 'uArchRate']\n",
    "    reduced_time = summary_df.at[replace_layer_type, 'tiuTime(us)'] * (1 - uArchRate / 100)\n",
    "    tiuTotalTime = tiuTotalTime - reduced_time\n",
    "    macUtil = tiuModelTheoTime / tiuTotalTime\n",
    "    return [replace_layer_type + f' uArchRate: {uArchRate:.2f}% -> 100%', reduced_time, tiuTotalTime, 100.00, macUtil*100, f'{replace_layer_type}的耗时用uArchRate=100%时的耗时替换']\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### macUtil分析总结\n",
    "由于呈现给大家的macUtil是通过tiuTheoTime / e2eTime, 此处共分析macUtil的四种影响因素：\n",
    "1. end2end -> origin：origin是profile中纯粹模型运行的耗时，这一步排除CPU耗时和输入输出的搬运耗时的影响，查看此时的macUtil\n",
    "2. origin -> 100% Concurrency：这一步排除并行度的干扰，即假设此时GDMA耗时被完全掩盖了，查看此时的macUtil\n",
    "3. 从100%并行度状态开始，依次将每种算子的tiuTime替换为tiuTheoTime（用整个模型的峰值算力计算的理论耗时），可以看出为什么macUtil无法达到100%\n",
    "4. 从100%并行度状态开始，依次替换每种算子的tiuTime为tiuPTheoTime（用该类型layer对应的峰值算力计算的理论耗时），可以看到当各个Layer替换为Profile中的理论耗时时，macUtil的提升程度\n",
    "5. 从100%并行度状态开始，依次将每种算子的uArchRate打满，看看当前计算方案下(这种情况与算子的实现方案强相关，可能存在更好的计算方案)，每种Layer的uArchRate打满时，macUtil的提升程度\n",
    "\n",
    "**注：**\n",
    "1. 由于每种算子的实现可能包含了其他指令，而这些指令并不能达到该算子的PeakTops，因此，通常即便uArchRate打满，其tiuTime依然会比tiuPTheoTime要长，体现在macUtil上就是macUtil很差，很难提升。\n",
    "2. 由于每条指令的算力不同，uArchRate打满情况的性能估算比较复杂，此处简单的采取tiuTime * uArchRate来估算打满时的性能，实际应当对每条指令这样计算打满的性能。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Matmul', 'Softmax', 'Cast', 'LayerNorm', 'Eltwise', 'Lut', 'Others']\n"
     ]
    }
   ],
   "source": [
    "row_names = summary_df.index.to_list()\n",
    "ops = []\n",
    "for op in row_names:\n",
    "  if op in ['Load', 'Store', 'Overall', 'Others']:\n",
    "    continue\n",
    "  ops.append(op)\n",
    "ops.append('Others')\n",
    "print(ops)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "columns = ['Case', 'ReducedTime(us)', 'Time(us)', 'Concurrency(%)', 'macUtil(%)', 'Remark']\n",
    "infosA = [\n",
    "    ['end2end', 0, e2eTime, summary_df.at['Overall', 'Concurrency'], macUtil0*100, '真正的macUtil'],\n",
    "    ['origin', e2eTime - totalModelTime, totalModelTime, summary_df.at['Overall', 'Concurrency'], macUtil1*100, '排除CPU耗时及输入输出在runtime空间和用户空间的搬运耗时'],\n",
    "    ['100% Concurrency', totalModelTime-tiuModelTime, tiuModelTime,  100.00, macUtil2*100, '排除并行度的干扰'],\n",
    "]\n",
    "\n",
    "infosB = []\n",
    "cur_time = tiuModelTime\n",
    "for op in ops:\n",
    "    infosB.append(get_mac_util1_with_tiu_theo_time(tiuModelTheoTime, cur_time, op))\n",
    "    cur_time = infosB[-1][2]\n",
    "\n",
    "infosC = []\n",
    "cur_time = tiuModelTime\n",
    "for op in ops:\n",
    "    infosC.append(get_mac_util2_with_tiu_ptheo_time(tiuModelTheoTime, cur_time, op))\n",
    "    cur_time = infosC[-1][2]\n",
    "\n",
    "infosD = []\n",
    "cur_time = tiuModelTime\n",
    "for op in ops:\n",
    "    infosD.append(get_mac_util3_with_full_uarch_rate(tiuModelTheoTime, cur_time, op))\n",
    "    cur_time = infosD[-1][2]\n",
    "\n",
    "\n",
    "\n",
    "infos = infosA + infosB + infosC + infosD\n",
    "df = pd.DataFrame(infos, columns = columns).round(2)\n",
    "\n",
    "# 保存到csv文件中\n",
    "df.to_csv(profile_dir + 'mac_util_analysis.csv', index=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Case</th>\n",
       "      <th>ReducedTime(us)</th>\n",
       "      <th>Time(us)</th>\n",
       "      <th>Concurrency(%)</th>\n",
       "      <th>macUtil(%)</th>\n",
       "      <th>Remark</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>end2end</td>\n",
       "      <td>0.00</td>\n",
       "      <td>38300.00</td>\n",
       "      <td>53.14</td>\n",
       "      <td>44.95</td>\n",
       "      <td>真正的macUtil</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>origin</td>\n",
       "      <td>598.95</td>\n",
       "      <td>37701.05</td>\n",
       "      <td>53.14</td>\n",
       "      <td>45.67</td>\n",
       "      <td>排除CPU耗时及输入输出在runtime空间和用户空间的搬运耗时</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>100% Concurrency</td>\n",
       "      <td>2926.52</td>\n",
       "      <td>34774.53</td>\n",
       "      <td>100.00</td>\n",
       "      <td>49.51</td>\n",
       "      <td>排除并行度的干扰</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Matmul tiuTime: 23054.57us -&gt; 17205.53us</td>\n",
       "      <td>5849.03</td>\n",
       "      <td>28925.50</td>\n",
       "      <td>100.00</td>\n",
       "      <td>59.52</td>\n",
       "      <td>Matmul的耗时用ModelPeakTops得到的理论耗时替换</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Softmax tiuTime: 6776.38us -&gt; 90.09us</td>\n",
       "      <td>6686.29</td>\n",
       "      <td>22239.21</td>\n",
       "      <td>100.00</td>\n",
       "      <td>77.42</td>\n",
       "      <td>Softmax的耗时用ModelPeakTops得到的理论耗时替换</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Cast tiuTime: 1364.05us -&gt; 32.83us</td>\n",
       "      <td>1331.22</td>\n",
       "      <td>20907.99</td>\n",
       "      <td>100.00</td>\n",
       "      <td>82.34</td>\n",
       "      <td>Cast的耗时用ModelPeakTops得到的理论耗时替换</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>LayerNorm tiuTime: 1495.61us -&gt; 19.20us</td>\n",
       "      <td>1476.42</td>\n",
       "      <td>19431.57</td>\n",
       "      <td>100.00</td>\n",
       "      <td>88.60</td>\n",
       "      <td>LayerNorm的耗时用ModelPeakTops得到的理论耗时替换</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Eltwise tiuTime: 110.16us -&gt; 6.43us</td>\n",
       "      <td>103.74</td>\n",
       "      <td>19327.83</td>\n",
       "      <td>100.00</td>\n",
       "      <td>89.08</td>\n",
       "      <td>Eltwise的耗时用ModelPeakTops得到的理论耗时替换</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Lut tiuTime: 1845.26us -&gt; 3.57us</td>\n",
       "      <td>1841.69</td>\n",
       "      <td>17486.15</td>\n",
       "      <td>100.00</td>\n",
       "      <td>98.46</td>\n",
       "      <td>Lut的耗时用ModelPeakTops得到的理论耗时替换</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Others tiuTime: 128.50us -&gt; 3.03us</td>\n",
       "      <td>125.47</td>\n",
       "      <td>17360.68</td>\n",
       "      <td>100.00</td>\n",
       "      <td>99.17</td>\n",
       "      <td>Others的耗时用ModelPeakTops得到的理论耗时替换</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Matmul tiuTime: 23054.57us -&gt; 17205.53us</td>\n",
       "      <td>5849.03</td>\n",
       "      <td>28925.50</td>\n",
       "      <td>100.00</td>\n",
       "      <td>59.52</td>\n",
       "      <td>Matmul的耗时用LayerPeakTops得到的理论耗时替换</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Softmax tiuTime: 6776.38us -&gt; 2883.02us</td>\n",
       "      <td>3893.37</td>\n",
       "      <td>25032.13</td>\n",
       "      <td>100.00</td>\n",
       "      <td>68.78</td>\n",
       "      <td>Softmax的耗时用LayerPeakTops得到的理论耗时替换</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Cast tiuTime: 1364.05us -&gt; 577.62us</td>\n",
       "      <td>786.43</td>\n",
       "      <td>24245.70</td>\n",
       "      <td>100.00</td>\n",
       "      <td>71.01</td>\n",
       "      <td>Cast的耗时用LayerPeakTops得到的理论耗时替换</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>LayerNorm tiuTime: 1495.61us -&gt; 307.12us</td>\n",
       "      <td>1188.49</td>\n",
       "      <td>23057.21</td>\n",
       "      <td>100.00</td>\n",
       "      <td>74.67</td>\n",
       "      <td>LayerNorm的耗时用LayerPeakTops得到的理论耗时替换</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Eltwise tiuTime: 110.16us -&gt; 51.42us</td>\n",
       "      <td>58.75</td>\n",
       "      <td>22998.46</td>\n",
       "      <td>100.00</td>\n",
       "      <td>74.86</td>\n",
       "      <td>Eltwise的耗时用LayerPeakTops得到的理论耗时替换</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>Lut tiuTime: 1845.26us -&gt; 1828.10us</td>\n",
       "      <td>17.16</td>\n",
       "      <td>22981.30</td>\n",
       "      <td>100.00</td>\n",
       "      <td>74.92</td>\n",
       "      <td>Lut的耗时用LayerPeakTops得到的理论耗时替换</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>Others tiuTime: 128.50us -&gt; 24.81us</td>\n",
       "      <td>103.69</td>\n",
       "      <td>22877.62</td>\n",
       "      <td>100.00</td>\n",
       "      <td>75.25</td>\n",
       "      <td>Others的耗时用LayerPeakTops得到的理论耗时替换</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>Matmul uArchRate: 90.52% -&gt; 100%</td>\n",
       "      <td>2185.57</td>\n",
       "      <td>32588.96</td>\n",
       "      <td>100.00</td>\n",
       "      <td>52.83</td>\n",
       "      <td>Matmul的耗时用uArchRate=100%时的耗时替换</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>Softmax uArchRate: 75.30% -&gt; 100%</td>\n",
       "      <td>1673.70</td>\n",
       "      <td>30915.26</td>\n",
       "      <td>100.00</td>\n",
       "      <td>55.69</td>\n",
       "      <td>Softmax的耗时用uArchRate=100%时的耗时替换</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>Cast uArchRate: 80.65% -&gt; 100%</td>\n",
       "      <td>264.00</td>\n",
       "      <td>30651.26</td>\n",
       "      <td>100.00</td>\n",
       "      <td>56.17</td>\n",
       "      <td>Cast的耗时用uArchRate=100%时的耗时替换</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>LayerNorm uArchRate: 81.44% -&gt; 100%</td>\n",
       "      <td>277.57</td>\n",
       "      <td>30373.69</td>\n",
       "      <td>100.00</td>\n",
       "      <td>56.68</td>\n",
       "      <td>LayerNorm的耗时用uArchRate=100%时的耗时替换</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>Eltwise uArchRate: 93.11% -&gt; 100%</td>\n",
       "      <td>7.59</td>\n",
       "      <td>30366.09</td>\n",
       "      <td>100.00</td>\n",
       "      <td>56.70</td>\n",
       "      <td>Eltwise的耗时用uArchRate=100%时的耗时替换</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>Lut uArchRate: 98.51% -&gt; 100%</td>\n",
       "      <td>27.49</td>\n",
       "      <td>30338.60</td>\n",
       "      <td>100.00</td>\n",
       "      <td>56.75</td>\n",
       "      <td>Lut的耗时用uArchRate=100%时的耗时替换</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>Others uArchRate: 77.56% -&gt; 100%</td>\n",
       "      <td>28.83</td>\n",
       "      <td>30309.77</td>\n",
       "      <td>100.00</td>\n",
       "      <td>56.80</td>\n",
       "      <td>Others的耗时用uArchRate=100%时的耗时替换</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                        Case  ReducedTime(us)  Time(us)  Concurrency(%)  macUtil(%)                               Remark\n",
       "0                                    end2end             0.00  38300.00           53.14       44.95                           真正的macUtil\n",
       "1                                     origin           598.95  37701.05           53.14       45.67     排除CPU耗时及输入输出在runtime空间和用户空间的搬运耗时\n",
       "2                           100% Concurrency          2926.52  34774.53          100.00       49.51                             排除并行度的干扰\n",
       "3   Matmul tiuTime: 23054.57us -> 17205.53us          5849.03  28925.50          100.00       59.52     Matmul的耗时用ModelPeakTops得到的理论耗时替换\n",
       "4      Softmax tiuTime: 6776.38us -> 90.09us          6686.29  22239.21          100.00       77.42    Softmax的耗时用ModelPeakTops得到的理论耗时替换\n",
       "5         Cast tiuTime: 1364.05us -> 32.83us          1331.22  20907.99          100.00       82.34       Cast的耗时用ModelPeakTops得到的理论耗时替换\n",
       "6    LayerNorm tiuTime: 1495.61us -> 19.20us          1476.42  19431.57          100.00       88.60  LayerNorm的耗时用ModelPeakTops得到的理论耗时替换\n",
       "7        Eltwise tiuTime: 110.16us -> 6.43us           103.74  19327.83          100.00       89.08    Eltwise的耗时用ModelPeakTops得到的理论耗时替换\n",
       "8           Lut tiuTime: 1845.26us -> 3.57us          1841.69  17486.15          100.00       98.46        Lut的耗时用ModelPeakTops得到的理论耗时替换\n",
       "9         Others tiuTime: 128.50us -> 3.03us           125.47  17360.68          100.00       99.17     Others的耗时用ModelPeakTops得到的理论耗时替换\n",
       "10  Matmul tiuTime: 23054.57us -> 17205.53us          5849.03  28925.50          100.00       59.52     Matmul的耗时用LayerPeakTops得到的理论耗时替换\n",
       "11   Softmax tiuTime: 6776.38us -> 2883.02us          3893.37  25032.13          100.00       68.78    Softmax的耗时用LayerPeakTops得到的理论耗时替换\n",
       "12       Cast tiuTime: 1364.05us -> 577.62us           786.43  24245.70          100.00       71.01       Cast的耗时用LayerPeakTops得到的理论耗时替换\n",
       "13  LayerNorm tiuTime: 1495.61us -> 307.12us          1188.49  23057.21          100.00       74.67  LayerNorm的耗时用LayerPeakTops得到的理论耗时替换\n",
       "14      Eltwise tiuTime: 110.16us -> 51.42us            58.75  22998.46          100.00       74.86    Eltwise的耗时用LayerPeakTops得到的理论耗时替换\n",
       "15       Lut tiuTime: 1845.26us -> 1828.10us            17.16  22981.30          100.00       74.92        Lut的耗时用LayerPeakTops得到的理论耗时替换\n",
       "16       Others tiuTime: 128.50us -> 24.81us           103.69  22877.62          100.00       75.25     Others的耗时用LayerPeakTops得到的理论耗时替换\n",
       "17          Matmul uArchRate: 90.52% -> 100%          2185.57  32588.96          100.00       52.83       Matmul的耗时用uArchRate=100%时的耗时替换\n",
       "18         Softmax uArchRate: 75.30% -> 100%          1673.70  30915.26          100.00       55.69      Softmax的耗时用uArchRate=100%时的耗时替换\n",
       "19            Cast uArchRate: 80.65% -> 100%           264.00  30651.26          100.00       56.17         Cast的耗时用uArchRate=100%时的耗时替换\n",
       "20       LayerNorm uArchRate: 81.44% -> 100%           277.57  30373.69          100.00       56.68    LayerNorm的耗时用uArchRate=100%时的耗时替换\n",
       "21         Eltwise uArchRate: 93.11% -> 100%             7.59  30366.09          100.00       56.70      Eltwise的耗时用uArchRate=100%时的耗时替换\n",
       "22             Lut uArchRate: 98.51% -> 100%            27.49  30338.60          100.00       56.75          Lut的耗时用uArchRate=100%时的耗时替换\n",
       "23          Others uArchRate: 77.56% -> 100%            28.83  30309.77          100.00       56.80       Others的耗时用uArchRate=100%时的耗时替换"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
